2024-11-24 09:43:17,613 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 09:43:17,623 main DEBUG Took 0.008727 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 09:43:17,624 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 09:43:17,624 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 09:43:17,625 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 09:43:17,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,633 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 09:43:17,644 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,645 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,646 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,646 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,646 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,647 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,648 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,648 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,648 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,649 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,649 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,650 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,650 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,650 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,651 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,651 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,652 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,652 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,652 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,653 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,653 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,653 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,654 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,654 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:43:17,655 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,655 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 09:43:17,657 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:43:17,658 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 09:43:17,659 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 09:43:17,660 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 09:43:17,661 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 09:43:17,661 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 09:43:17,669 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 09:43:17,671 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 09:43:17,673 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 09:43:17,673 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 09:43:17,673 main DEBUG createAppenders(={Console}) 2024-11-24 09:43:17,674 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-24 09:43:17,674 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 09:43:17,675 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-24 09:43:17,675 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 09:43:17,676 main DEBUG OutputStream closed 2024-11-24 09:43:17,676 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 09:43:17,676 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 09:43:17,677 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-24 09:43:17,761 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 09:43:17,764 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 09:43:17,765 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 09:43:17,766 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 09:43:17,767 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 09:43:17,767 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 09:43:17,768 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 09:43:17,768 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 09:43:17,769 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 09:43:17,769 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 09:43:17,770 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 09:43:17,770 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 09:43:17,770 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 09:43:17,771 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 09:43:17,771 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 09:43:17,771 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 09:43:17,772 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 09:43:17,772 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 09:43:17,775 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 09:43:17,775 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-24 09:43:17,776 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 09:43:17,777 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-24T09:43:17,996 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb 2024-11-24 09:43:17,999 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 09:43:17,999 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T09:43:18,007 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-24T09:43:18,013 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-11-24T09:43:18,034 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T09:43:18,074 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T09:43:18,074 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T09:43:18,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T09:43:18,098 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779, deleteOnExit=true 2024-11-24T09:43:18,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T09:43:18,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/test.cache.data in system properties and HBase conf 2024-11-24T09:43:18,100 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T09:43:18,100 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/hadoop.log.dir in system properties and HBase conf 2024-11-24T09:43:18,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T09:43:18,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T09:43:18,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T09:43:18,170 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T09:43:18,250 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T09:43:18,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T09:43:18,254 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T09:43:18,260 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T09:43:18,260 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T09:43:18,261 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T09:43:18,261 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T09:43:18,261 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T09:43:18,262 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T09:43:18,262 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T09:43:18,263 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/nfs.dump.dir in system properties and HBase conf 2024-11-24T09:43:18,263 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/java.io.tmpdir in system properties and HBase conf 2024-11-24T09:43:18,263 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T09:43:18,264 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T09:43:18,264 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T09:43:19,225 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T09:43:19,301 INFO [Time-limited test {}] log.Log(170): Logging initialized @2339ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T09:43:19,366 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T09:43:19,420 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T09:43:19,439 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T09:43:19,439 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T09:43:19,441 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T09:43:19,457 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T09:43:19,460 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/hadoop.log.dir/,AVAILABLE} 2024-11-24T09:43:19,460 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T09:43:19,640 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/java.io.tmpdir/jetty-localhost-37645-hadoop-hdfs-3_4_1-tests_jar-_-any-14779614602172859058/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T09:43:19,645 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:37645} 2024-11-24T09:43:19,645 INFO [Time-limited test {}] server.Server(415): Started @2684ms 2024-11-24T09:43:20,117 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T09:43:20,124 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T09:43:20,125 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T09:43:20,125 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T09:43:20,125 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T09:43:20,126 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/hadoop.log.dir/,AVAILABLE} 2024-11-24T09:43:20,126 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T09:43:20,225 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/java.io.tmpdir/jetty-localhost-45415-hadoop-hdfs-3_4_1-tests_jar-_-any-826918661615039498/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T09:43:20,226 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:45415} 2024-11-24T09:43:20,226 INFO [Time-limited test {}] server.Server(415): Started @3264ms 2024-11-24T09:43:20,275 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T09:43:20,380 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T09:43:20,385 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T09:43:20,391 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T09:43:20,391 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T09:43:20,391 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T09:43:20,392 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/hadoop.log.dir/,AVAILABLE} 2024-11-24T09:43:20,393 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T09:43:20,502 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/java.io.tmpdir/jetty-localhost-44531-hadoop-hdfs-3_4_1-tests_jar-_-any-16397465344509074629/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T09:43:20,503 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:44531} 2024-11-24T09:43:20,503 INFO [Time-limited test {}] server.Server(415): Started @3542ms 2024-11-24T09:43:20,505 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T09:43:20,543 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T09:43:20,550 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T09:43:20,555 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T09:43:20,555 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T09:43:20,555 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T09:43:20,556 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/hadoop.log.dir/,AVAILABLE} 2024-11-24T09:43:20,557 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T09:43:20,659 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/java.io.tmpdir/jetty-localhost-34259-hadoop-hdfs-3_4_1-tests_jar-_-any-17358841471311755338/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T09:43:20,660 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:34259} 2024-11-24T09:43:20,660 INFO [Time-limited test {}] server.Server(415): Started @3698ms 2024-11-24T09:43:20,662 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T09:43:21,398 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data3/current/BP-880808591-172.17.0.2-1732441398809/current, will proceed with Du for space computation calculation, 2024-11-24T09:43:21,398 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data1/current/BP-880808591-172.17.0.2-1732441398809/current, will proceed with Du for space computation calculation, 2024-11-24T09:43:21,398 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data2/current/BP-880808591-172.17.0.2-1732441398809/current, will proceed with Du for space computation calculation, 2024-11-24T09:43:21,398 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data4/current/BP-880808591-172.17.0.2-1732441398809/current, will proceed with Du for space computation calculation, 2024-11-24T09:43:21,431 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T09:43:21,431 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T09:43:21,474 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5962bc1518c7a481 with lease ID 0x3d389ddc3cd93884: Processing first storage report for DS-cece125c-6b20-4102-926b-c1801fb0cd76 from datanode DatanodeRegistration(127.0.0.1:33653, datanodeUuid=9fa7c2d1-dbb5-474d-bb12-c3da4aaa534a, infoPort=40591, infoSecurePort=0, ipcPort=41089, storageInfo=lv=-57;cid=testClusterID;nsid=561316763;c=1732441398809) 2024-11-24T09:43:21,475 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5962bc1518c7a481 with lease ID 0x3d389ddc3cd93884: from storage DS-cece125c-6b20-4102-926b-c1801fb0cd76 node DatanodeRegistration(127.0.0.1:33653, datanodeUuid=9fa7c2d1-dbb5-474d-bb12-c3da4aaa534a, infoPort=40591, infoSecurePort=0, ipcPort=41089, storageInfo=lv=-57;cid=testClusterID;nsid=561316763;c=1732441398809), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T09:43:21,475 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5c16e17242426bcf with lease ID 0x3d389ddc3cd93885: Processing first storage report for DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4 from datanode DatanodeRegistration(127.0.0.1:37499, datanodeUuid=9024e8f8-076f-4b78-a55d-0d0a87f756bf, infoPort=42101, infoSecurePort=0, ipcPort=43857, storageInfo=lv=-57;cid=testClusterID;nsid=561316763;c=1732441398809) 2024-11-24T09:43:21,476 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5c16e17242426bcf with lease ID 0x3d389ddc3cd93885: from storage DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4 node DatanodeRegistration(127.0.0.1:37499, datanodeUuid=9024e8f8-076f-4b78-a55d-0d0a87f756bf, infoPort=42101, infoSecurePort=0, ipcPort=43857, storageInfo=lv=-57;cid=testClusterID;nsid=561316763;c=1732441398809), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T09:43:21,476 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5962bc1518c7a481 with lease ID 0x3d389ddc3cd93884: Processing first storage report for DS-98c384e3-4fa2-4edb-ac66-9f5e58a2b292 from datanode DatanodeRegistration(127.0.0.1:33653, datanodeUuid=9fa7c2d1-dbb5-474d-bb12-c3da4aaa534a, infoPort=40591, infoSecurePort=0, ipcPort=41089, storageInfo=lv=-57;cid=testClusterID;nsid=561316763;c=1732441398809) 2024-11-24T09:43:21,476 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5962bc1518c7a481 with lease ID 0x3d389ddc3cd93884: from storage DS-98c384e3-4fa2-4edb-ac66-9f5e58a2b292 node DatanodeRegistration(127.0.0.1:33653, datanodeUuid=9fa7c2d1-dbb5-474d-bb12-c3da4aaa534a, infoPort=40591, infoSecurePort=0, ipcPort=41089, storageInfo=lv=-57;cid=testClusterID;nsid=561316763;c=1732441398809), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T09:43:21,476 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5c16e17242426bcf with lease ID 0x3d389ddc3cd93885: Processing first storage report for DS-eeef8872-c4c6-47c8-b7dc-66b7ceb12871 from datanode DatanodeRegistration(127.0.0.1:37499, datanodeUuid=9024e8f8-076f-4b78-a55d-0d0a87f756bf, infoPort=42101, infoSecurePort=0, ipcPort=43857, storageInfo=lv=-57;cid=testClusterID;nsid=561316763;c=1732441398809) 2024-11-24T09:43:21,476 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5c16e17242426bcf with lease ID 0x3d389ddc3cd93885: from storage DS-eeef8872-c4c6-47c8-b7dc-66b7ceb12871 node DatanodeRegistration(127.0.0.1:37499, datanodeUuid=9024e8f8-076f-4b78-a55d-0d0a87f756bf, infoPort=42101, infoSecurePort=0, ipcPort=43857, storageInfo=lv=-57;cid=testClusterID;nsid=561316763;c=1732441398809), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T09:43:21,499 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data6/current/BP-880808591-172.17.0.2-1732441398809/current, will proceed with Du for space computation calculation, 2024-11-24T09:43:21,499 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data5/current/BP-880808591-172.17.0.2-1732441398809/current, will proceed with Du for space computation calculation, 2024-11-24T09:43:21,518 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T09:43:21,523 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa98771794b3b31a9 with lease ID 0x3d389ddc3cd93886: Processing first storage report for DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977 from datanode DatanodeRegistration(127.0.0.1:37953, datanodeUuid=27220eb0-c1fd-4d9d-ab92-71a0aab887ed, infoPort=35185, infoSecurePort=0, ipcPort=38289, storageInfo=lv=-57;cid=testClusterID;nsid=561316763;c=1732441398809) 2024-11-24T09:43:21,523 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa98771794b3b31a9 with lease ID 0x3d389ddc3cd93886: from storage DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977 node DatanodeRegistration(127.0.0.1:37953, datanodeUuid=27220eb0-c1fd-4d9d-ab92-71a0aab887ed, infoPort=35185, infoSecurePort=0, ipcPort=38289, storageInfo=lv=-57;cid=testClusterID;nsid=561316763;c=1732441398809), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T09:43:21,523 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa98771794b3b31a9 with lease ID 0x3d389ddc3cd93886: Processing first storage report for DS-7e2f81aa-c261-46a1-b1cf-aba14d217b13 from datanode DatanodeRegistration(127.0.0.1:37953, datanodeUuid=27220eb0-c1fd-4d9d-ab92-71a0aab887ed, infoPort=35185, infoSecurePort=0, ipcPort=38289, storageInfo=lv=-57;cid=testClusterID;nsid=561316763;c=1732441398809) 2024-11-24T09:43:21,524 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa98771794b3b31a9 with lease ID 0x3d389ddc3cd93886: from storage DS-7e2f81aa-c261-46a1-b1cf-aba14d217b13 node DatanodeRegistration(127.0.0.1:37953, datanodeUuid=27220eb0-c1fd-4d9d-ab92-71a0aab887ed, infoPort=35185, infoSecurePort=0, ipcPort=38289, storageInfo=lv=-57;cid=testClusterID;nsid=561316763;c=1732441398809), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T09:43:21,570 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb 2024-11-24T09:43:21,627 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/zookeeper_0, clientPort=57552, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T09:43:21,636 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57552 2024-11-24T09:43:21,648 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:43:21,650 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:43:21,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741825_1001 (size=7) 2024-11-24T09:43:21,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741825_1001 (size=7) 2024-11-24T09:43:21,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741825_1001 (size=7) 2024-11-24T09:43:22,244 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88 with version=8 2024-11-24T09:43:22,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/hbase-staging 2024-11-24T09:43:22,523 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ba35ada4783f:0 server-side Connection retries=45 2024-11-24T09:43:22,532 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:43:22,532 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T09:43:22,536 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T09:43:22,536 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:43:22,537 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T09:43:22,658 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T09:43:22,712 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T09:43:22,721 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T09:43:22,724 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T09:43:22,748 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 9588 (auto-detected) 2024-11-24T09:43:22,749 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T09:43:22,767 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39325 2024-11-24T09:43:22,787 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39325 connecting to ZooKeeper ensemble=127.0.0.1:57552 2024-11-24T09:43:22,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:393250x0, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T09:43:22,901 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39325-0x1016d837bd10000 connected 2024-11-24T09:43:22,990 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:43:22,997 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:43:23,009 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:43:23,013 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88, hbase.cluster.distributed=false 2024-11-24T09:43:23,033 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T09:43:23,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39325 2024-11-24T09:43:23,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39325 2024-11-24T09:43:23,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39325 2024-11-24T09:43:23,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39325 2024-11-24T09:43:23,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39325 2024-11-24T09:43:23,141 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba35ada4783f:0 server-side Connection retries=45 2024-11-24T09:43:23,143 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:43:23,143 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T09:43:23,143 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T09:43:23,144 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:43:23,144 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T09:43:23,146 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T09:43:23,149 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T09:43:23,150 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42075 2024-11-24T09:43:23,152 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42075 connecting to ZooKeeper ensemble=127.0.0.1:57552 2024-11-24T09:43:23,154 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:43:23,157 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:43:23,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:420750x0, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T09:43:23,196 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:420750x0, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:43:23,196 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42075-0x1016d837bd10001 connected 2024-11-24T09:43:23,200 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T09:43:23,208 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T09:43:23,211 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T09:43:23,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T09:43:23,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42075 2024-11-24T09:43:23,219 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42075 2024-11-24T09:43:23,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42075 2024-11-24T09:43:23,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42075 2024-11-24T09:43:23,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42075 2024-11-24T09:43:23,238 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba35ada4783f:0 server-side Connection retries=45 2024-11-24T09:43:23,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:43:23,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T09:43:23,239 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T09:43:23,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:43:23,240 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T09:43:23,240 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T09:43:23,240 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T09:43:23,241 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34867 2024-11-24T09:43:23,243 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34867 connecting to ZooKeeper ensemble=127.0.0.1:57552 2024-11-24T09:43:23,244 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:43:23,246 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:43:23,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348670x0, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T09:43:23,270 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:43:23,270 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34867-0x1016d837bd10002 connected 2024-11-24T09:43:23,271 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T09:43:23,272 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T09:43:23,273 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T09:43:23,275 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T09:43:23,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34867 2024-11-24T09:43:23,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34867 2024-11-24T09:43:23,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34867 2024-11-24T09:43:23,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34867 2024-11-24T09:43:23,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34867 2024-11-24T09:43:23,296 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba35ada4783f:0 server-side Connection retries=45 2024-11-24T09:43:23,296 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:43:23,296 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T09:43:23,296 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T09:43:23,296 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:43:23,296 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T09:43:23,297 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T09:43:23,297 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T09:43:23,298 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46517 2024-11-24T09:43:23,299 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46517 connecting to ZooKeeper ensemble=127.0.0.1:57552 2024-11-24T09:43:23,300 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:43:23,302 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:43:23,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:465170x0, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T09:43:23,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:465170x0, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:43:23,330 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46517-0x1016d837bd10003 connected 2024-11-24T09:43:23,331 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T09:43:23,332 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T09:43:23,333 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T09:43:23,336 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T09:43:23,337 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46517 2024-11-24T09:43:23,337 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46517 2024-11-24T09:43:23,338 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46517 2024-11-24T09:43:23,338 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46517 2024-11-24T09:43:23,338 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46517 2024-11-24T09:43:23,354 DEBUG [M:0;ba35ada4783f:39325 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ba35ada4783f:39325 2024-11-24T09:43:23,355 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ba35ada4783f,39325,1732441402377 2024-11-24T09:43:23,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:43:23,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:43:23,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:43:23,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:43:23,372 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ba35ada4783f,39325,1732441402377 2024-11-24T09:43:23,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T09:43:23,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T09:43:23,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:23,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:23,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:23,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T09:43:23,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:23,396 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T09:43:23,397 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ba35ada4783f,39325,1732441402377 from backup master directory 2024-11-24T09:43:23,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ba35ada4783f,39325,1732441402377 2024-11-24T09:43:23,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:43:23,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:43:23,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:43:23,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:43:23,404 WARN [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T09:43:23,404 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ba35ada4783f,39325,1732441402377 2024-11-24T09:43:23,406 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T09:43:23,408 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T09:43:23,458 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/hbase.id] with ID: 574c19c5-3bf5-487a-a197-015a772295c0 2024-11-24T09:43:23,459 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/.tmp/hbase.id 2024-11-24T09:43:23,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741826_1002 (size=42) 2024-11-24T09:43:23,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741826_1002 (size=42) 2024-11-24T09:43:23,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741826_1002 (size=42) 2024-11-24T09:43:23,474 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/.tmp/hbase.id]:[hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/hbase.id] 2024-11-24T09:43:23,515 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:43:23,519 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T09:43:23,537 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-24T09:43:23,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:23,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:23,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:23,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:23,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741827_1003 (size=196) 2024-11-24T09:43:23,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741827_1003 (size=196) 2024-11-24T09:43:23,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741827_1003 (size=196) 2024-11-24T09:43:23,597 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T09:43:23,598 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T09:43:23,603 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:23,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741828_1004 (size=1189) 2024-11-24T09:43:23,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741828_1004 (size=1189) 2024-11-24T09:43:23,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741828_1004 (size=1189) 2024-11-24T09:43:23,651 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store 2024-11-24T09:43:23,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741829_1005 (size=34) 2024-11-24T09:43:23,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741829_1005 (size=34) 2024-11-24T09:43:23,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741829_1005 (size=34) 2024-11-24T09:43:23,679 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T09:43:23,682 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:23,683 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T09:43:23,683 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:43:23,683 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:43:23,685 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T09:43:23,685 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:43:23,685 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:43:23,686 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732441403683Disabling compacts and flushes for region at 1732441403683Disabling writes for close at 1732441403685 (+2 ms)Writing region close event to WAL at 1732441403685Closed at 1732441403685 2024-11-24T09:43:23,688 WARN [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/.initializing 2024-11-24T09:43:23,688 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/WALs/ba35ada4783f,39325,1732441402377 2024-11-24T09:43:23,695 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:23,713 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C39325%2C1732441402377, suffix=, logDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/WALs/ba35ada4783f,39325,1732441402377, archiveDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/oldWALs, maxLogs=10 2024-11-24T09:43:23,746 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/WALs/ba35ada4783f,39325,1732441402377/ba35ada4783f%2C39325%2C1732441402377.1732441403718, exclude list is [], retry=0 2024-11-24T09:43:23,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:23,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:23,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:23,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:23,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T09:43:23,804 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/WALs/ba35ada4783f,39325,1732441402377/ba35ada4783f%2C39325%2C1732441402377.1732441403718 2024-11-24T09:43:23,805 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:23,805 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:23,805 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:23,808 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:43:23,809 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:43:23,843 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:43:23,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T09:43:23,869 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:23,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:23,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:43:23,875 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T09:43:23,875 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:23,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:23,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:43:23,879 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T09:43:23,879 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:23,880 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:23,880 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:43:23,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T09:43:23,883 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:23,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:23,884 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:43:23,887 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:43:23,888 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:43:23,893 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:43:23,894 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:43:23,897 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T09:43:23,901 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:43:23,905 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:23,906 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60506448, jitterRate=-0.09838366508483887}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T09:43:23,914 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732441403820Initializing all the Stores at 1732441403822 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441403823 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441403824 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441403824Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441403824Cleaning up temporary data from old regions at 1732441403894 (+70 ms)Region opened successfully at 1732441403914 (+20 ms) 2024-11-24T09:43:23,915 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T09:43:23,946 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13347294, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba35ada4783f/172.17.0.2:0 2024-11-24T09:43:23,973 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T09:43:23,982 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T09:43:23,983 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T09:43:23,985 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T09:43:23,986 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T09:43:23,990 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T09:43:23,990 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T09:43:24,015 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T09:43:24,023 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T09:43:24,078 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T09:43:24,081 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T09:43:24,083 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T09:43:24,094 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T09:43:24,095 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T09:43:24,099 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T09:43:24,103 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T09:43:24,105 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T09:43:24,111 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T09:43:24,130 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T09:43:24,136 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T09:43:24,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T09:43:24,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T09:43:24,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T09:43:24,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T09:43:24,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:24,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:24,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:24,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:24,149 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ba35ada4783f,39325,1732441402377, sessionid=0x1016d837bd10000, setting cluster-up flag (Was=false) 2024-11-24T09:43:24,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:24,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:24,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:24,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:24,204 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T09:43:24,208 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba35ada4783f,39325,1732441402377 2024-11-24T09:43:24,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:24,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:24,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:24,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:24,253 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T09:43:24,255 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba35ada4783f,39325,1732441402377 2024-11-24T09:43:24,264 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T09:43:24,342 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(746): ClusterId : 574c19c5-3bf5-487a-a197-015a772295c0 2024-11-24T09:43:24,342 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(746): ClusterId : 574c19c5-3bf5-487a-a197-015a772295c0 2024-11-24T09:43:24,343 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(746): ClusterId : 574c19c5-3bf5-487a-a197-015a772295c0 2024-11-24T09:43:24,345 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T09:43:24,345 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T09:43:24,345 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T09:43:24,345 INFO [AsyncFSWAL-0-hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData-prefix:ba35ada4783f,39325,1732441402377 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-11-24T09:43:24,358 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T09:43:24,366 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T09:43:24,372 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T09:43:24,372 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T09:43:24,372 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T09:43:24,372 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T09:43:24,372 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T09:43:24,372 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T09:43:24,373 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T09:43:24,378 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ba35ada4783f,39325,1732441402377 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T09:43:24,387 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T09:43:24,387 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T09:43:24,388 DEBUG [RS:2;ba35ada4783f:46517 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77be3829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba35ada4783f/172.17.0.2:0 2024-11-24T09:43:24,388 DEBUG [RS:1;ba35ada4783f:34867 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@304408c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba35ada4783f/172.17.0.2:0 2024-11-24T09:43:24,388 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T09:43:24,389 DEBUG [RS:0;ba35ada4783f:42075 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c7b8db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba35ada4783f/172.17.0.2:0 2024-11-24T09:43:24,389 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ba35ada4783f:0, corePoolSize=5, maxPoolSize=5 2024-11-24T09:43:24,389 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ba35ada4783f:0, corePoolSize=5, maxPoolSize=5 2024-11-24T09:43:24,390 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ba35ada4783f:0, corePoolSize=5, maxPoolSize=5 2024-11-24T09:43:24,390 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ba35ada4783f:0, corePoolSize=5, maxPoolSize=5 2024-11-24T09:43:24,390 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ba35ada4783f:0, corePoolSize=10, maxPoolSize=10 2024-11-24T09:43:24,390 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,390 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ba35ada4783f:0, corePoolSize=2, maxPoolSize=2 2024-11-24T09:43:24,390 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,398 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T09:43:24,398 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T09:43:24,399 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732441434399 2024-11-24T09:43:24,401 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T09:43:24,402 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T09:43:24,405 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:24,405 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T09:43:24,406 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T09:43:24,407 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T09:43:24,407 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T09:43:24,407 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T09:43:24,407 DEBUG [RS:0;ba35ada4783f:42075 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ba35ada4783f:42075 2024-11-24T09:43:24,407 DEBUG [RS:2;ba35ada4783f:46517 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ba35ada4783f:46517 2024-11-24T09:43:24,407 DEBUG [RS:1;ba35ada4783f:34867 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ba35ada4783f:34867 2024-11-24T09:43:24,411 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T09:43:24,411 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T09:43:24,411 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T09:43:24,411 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T09:43:24,411 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T09:43:24,411 DEBUG [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T09:43:24,411 DEBUG [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T09:43:24,411 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T09:43:24,412 DEBUG [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T09:43:24,411 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,414 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T09:43:24,414 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba35ada4783f,39325,1732441402377 with port=46517, startcode=1732441403295 2024-11-24T09:43:24,414 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba35ada4783f,39325,1732441402377 with port=42075, startcode=1732441403106 2024-11-24T09:43:24,414 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba35ada4783f,39325,1732441402377 with port=34867, startcode=1732441403238 2024-11-24T09:43:24,415 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T09:43:24,416 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T09:43:24,424 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T09:43:24,424 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T09:43:24,425 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.large.0-1732441404425,5,FailOnTimeoutGroup] 2024-11-24T09:43:24,426 DEBUG [RS:2;ba35ada4783f:46517 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T09:43:24,426 DEBUG [RS:0;ba35ada4783f:42075 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T09:43:24,426 DEBUG [RS:1;ba35ada4783f:34867 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T09:43:24,426 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.small.0-1732441404426,5,FailOnTimeoutGroup] 2024-11-24T09:43:24,427 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,427 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T09:43:24,428 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,428 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741831_1007 (size=1321) 2024-11-24T09:43:24,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741831_1007 (size=1321) 2024-11-24T09:43:24,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741831_1007 (size=1321) 2024-11-24T09:43:24,440 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T09:43:24,440 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88 2024-11-24T09:43:24,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741832_1008 (size=32) 2024-11-24T09:43:24,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741832_1008 (size=32) 2024-11-24T09:43:24,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741832_1008 (size=32) 2024-11-24T09:43:24,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:24,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T09:43:24,473 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T09:43:24,474 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:24,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:24,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T09:43:24,479 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T09:43:24,480 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:24,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:24,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T09:43:24,485 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T09:43:24,485 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:24,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:24,487 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40129, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T09:43:24,487 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50945, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T09:43:24,487 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53249, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T09:43:24,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T09:43:24,491 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T09:43:24,491 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:24,492 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:24,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T09:43:24,494 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740 2024-11-24T09:43:24,494 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba35ada4783f,46517,1732441403295 2024-11-24T09:43:24,495 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740 2024-11-24T09:43:24,498 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39325 {}] master.ServerManager(517): Registering regionserver=ba35ada4783f,46517,1732441403295 2024-11-24T09:43:24,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T09:43:24,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T09:43:24,499 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T09:43:24,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T09:43:24,508 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:24,509 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73434656, jitterRate=0.09426164627075195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T09:43:24,512 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba35ada4783f,42075,1732441403106 2024-11-24T09:43:24,512 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39325 {}] master.ServerManager(517): Registering regionserver=ba35ada4783f,42075,1732441403106 2024-11-24T09:43:24,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732441404466Initializing all the Stores at 1732441404468 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441404468Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441404469 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441404469Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441404469Cleaning up temporary data from old regions at 1732441404498 (+29 ms)Region opened successfully at 1732441404515 (+17 ms) 2024-11-24T09:43:24,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T09:43:24,516 DEBUG [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88 2024-11-24T09:43:24,516 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T09:43:24,516 DEBUG [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36549 2024-11-24T09:43:24,516 DEBUG [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T09:43:24,516 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T09:43:24,516 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T09:43:24,516 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T09:43:24,517 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba35ada4783f,34867,1732441403238 2024-11-24T09:43:24,517 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T09:43:24,517 DEBUG [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88 2024-11-24T09:43:24,518 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39325 {}] master.ServerManager(517): Registering regionserver=ba35ada4783f,34867,1732441403238 2024-11-24T09:43:24,518 DEBUG [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36549 2024-11-24T09:43:24,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732441404515Disabling compacts and flushes for region at 1732441404515Disabling writes for close at 1732441404516 (+1 ms)Writing region close event to WAL at 1732441404517 (+1 ms)Closed at 1732441404517 2024-11-24T09:43:24,518 DEBUG [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T09:43:24,521 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T09:43:24,521 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T09:43:24,522 DEBUG [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88 2024-11-24T09:43:24,522 DEBUG [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36549 2024-11-24T09:43:24,522 DEBUG [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T09:43:24,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T09:43:24,540 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T09:43:24,543 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T09:43:24,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T09:43:24,578 DEBUG [RS:0;ba35ada4783f:42075 {}] zookeeper.ZKUtil(111): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba35ada4783f,42075,1732441403106 2024-11-24T09:43:24,578 DEBUG [RS:2;ba35ada4783f:46517 {}] zookeeper.ZKUtil(111): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba35ada4783f,46517,1732441403295 2024-11-24T09:43:24,579 WARN [RS:0;ba35ada4783f:42075 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T09:43:24,579 WARN [RS:2;ba35ada4783f:46517 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T09:43:24,579 INFO [RS:0;ba35ada4783f:42075 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:24,579 INFO [RS:2;ba35ada4783f:46517 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:24,579 DEBUG [RS:1;ba35ada4783f:34867 {}] zookeeper.ZKUtil(111): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba35ada4783f,34867,1732441403238 2024-11-24T09:43:24,579 WARN [RS:1;ba35ada4783f:34867 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T09:43:24,579 DEBUG [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,42075,1732441403106 2024-11-24T09:43:24,579 DEBUG [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295 2024-11-24T09:43:24,579 INFO [RS:1;ba35ada4783f:34867 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:24,579 DEBUG [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,34867,1732441403238 2024-11-24T09:43:24,580 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba35ada4783f,34867,1732441403238] 2024-11-24T09:43:24,581 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba35ada4783f,46517,1732441403295] 2024-11-24T09:43:24,581 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba35ada4783f,42075,1732441403106] 2024-11-24T09:43:24,604 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T09:43:24,604 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T09:43:24,604 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T09:43:24,620 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T09:43:24,620 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T09:43:24,621 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T09:43:24,626 INFO [RS:2;ba35ada4783f:46517 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T09:43:24,626 INFO [RS:1;ba35ada4783f:34867 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T09:43:24,626 INFO [RS:0;ba35ada4783f:42075 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T09:43:24,626 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,626 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,626 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,627 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T09:43:24,627 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T09:43:24,630 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T09:43:24,633 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T09:43:24,633 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T09:43:24,633 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T09:43:24,635 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,635 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,635 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,635 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,635 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,635 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0, corePoolSize=2, maxPoolSize=2 2024-11-24T09:43:24,636 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0, corePoolSize=2, maxPoolSize=2 2024-11-24T09:43:24,636 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0, corePoolSize=2, maxPoolSize=2 2024-11-24T09:43:24,636 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,636 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:43:24,637 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:43:24,637 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:43:24,637 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:43:24,638 DEBUG [RS:0;ba35ada4783f:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:43:24,638 DEBUG [RS:2;ba35ada4783f:46517 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:43:24,638 DEBUG [RS:1;ba35ada4783f:34867 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:43:24,642 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,642 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,642 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,642 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,642 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,642 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,642 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,46517,1732441403295-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T09:43:24,642 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,643 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,643 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,643 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,643 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,34867,1732441403238-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T09:43:24,643 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,643 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,643 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,643 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,643 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,644 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,42075,1732441403106-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T09:43:24,664 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T09:43:24,665 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T09:43:24,665 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T09:43:24,666 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,42075,1732441403106-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,666 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,46517,1732441403295-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,666 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,34867,1732441403238-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,667 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,667 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,667 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,667 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.Replication(171): ba35ada4783f,46517,1732441403295 started 2024-11-24T09:43:24,667 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.Replication(171): ba35ada4783f,34867,1732441403238 started 2024-11-24T09:43:24,667 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.Replication(171): ba35ada4783f,42075,1732441403106 started 2024-11-24T09:43:24,684 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,684 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(1482): Serving as ba35ada4783f,42075,1732441403106, RpcServer on ba35ada4783f/172.17.0.2:42075, sessionid=0x1016d837bd10001 2024-11-24T09:43:24,685 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T09:43:24,685 DEBUG [RS:0;ba35ada4783f:42075 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba35ada4783f,42075,1732441403106 2024-11-24T09:43:24,685 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,42075,1732441403106' 2024-11-24T09:43:24,685 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T09:43:24,686 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T09:43:24,687 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T09:43:24,687 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T09:43:24,687 DEBUG [RS:0;ba35ada4783f:42075 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba35ada4783f,42075,1732441403106 2024-11-24T09:43:24,687 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,42075,1732441403106' 2024-11-24T09:43:24,687 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T09:43:24,687 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T09:43:24,688 DEBUG [RS:0;ba35ada4783f:42075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T09:43:24,688 INFO [RS:0;ba35ada4783f:42075 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T09:43:24,688 INFO [RS:0;ba35ada4783f:42075 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T09:43:24,690 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,690 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:24,690 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(1482): Serving as ba35ada4783f,46517,1732441403295, RpcServer on ba35ada4783f/172.17.0.2:46517, sessionid=0x1016d837bd10003 2024-11-24T09:43:24,690 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(1482): Serving as ba35ada4783f,34867,1732441403238, RpcServer on ba35ada4783f/172.17.0.2:34867, sessionid=0x1016d837bd10002 2024-11-24T09:43:24,690 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T09:43:24,690 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T09:43:24,690 DEBUG [RS:2;ba35ada4783f:46517 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba35ada4783f,46517,1732441403295 2024-11-24T09:43:24,690 DEBUG [RS:1;ba35ada4783f:34867 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba35ada4783f,34867,1732441403238 2024-11-24T09:43:24,691 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,46517,1732441403295' 2024-11-24T09:43:24,691 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,34867,1732441403238' 2024-11-24T09:43:24,691 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T09:43:24,691 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T09:43:24,691 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T09:43:24,691 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T09:43:24,692 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T09:43:24,692 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T09:43:24,692 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T09:43:24,692 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T09:43:24,692 DEBUG [RS:1;ba35ada4783f:34867 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba35ada4783f,34867,1732441403238 2024-11-24T09:43:24,692 DEBUG [RS:2;ba35ada4783f:46517 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba35ada4783f,46517,1732441403295 2024-11-24T09:43:24,692 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,34867,1732441403238' 2024-11-24T09:43:24,692 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,46517,1732441403295' 2024-11-24T09:43:24,692 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T09:43:24,692 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T09:43:24,693 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T09:43:24,693 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T09:43:24,693 DEBUG [RS:1;ba35ada4783f:34867 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T09:43:24,693 DEBUG [RS:2;ba35ada4783f:46517 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T09:43:24,693 INFO [RS:1;ba35ada4783f:34867 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T09:43:24,693 INFO [RS:2;ba35ada4783f:46517 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T09:43:24,693 INFO [RS:1;ba35ada4783f:34867 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T09:43:24,693 INFO [RS:2;ba35ada4783f:46517 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T09:43:24,694 WARN [ba35ada4783f:39325 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T09:43:24,795 INFO [RS:1;ba35ada4783f:34867 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:24,795 INFO [RS:0;ba35ada4783f:42075 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:24,795 INFO [RS:2;ba35ada4783f:46517 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:24,798 INFO [RS:0;ba35ada4783f:42075 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C42075%2C1732441403106, suffix=, logDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,42075,1732441403106, archiveDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/oldWALs, maxLogs=32 2024-11-24T09:43:24,798 INFO [RS:1;ba35ada4783f:34867 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C34867%2C1732441403238, suffix=, logDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,34867,1732441403238, archiveDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/oldWALs, maxLogs=32 2024-11-24T09:43:24,798 INFO [RS:2;ba35ada4783f:46517 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C46517%2C1732441403295, suffix=, logDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295, archiveDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/oldWALs, maxLogs=32 2024-11-24T09:43:24,815 DEBUG [RS:1;ba35ada4783f:34867 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,34867,1732441403238/ba35ada4783f%2C34867%2C1732441403238.1732441404802, exclude list is [], retry=0 2024-11-24T09:43:24,815 DEBUG [RS:0;ba35ada4783f:42075 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,42075,1732441403106/ba35ada4783f%2C42075%2C1732441403106.1732441404802, exclude list is [], retry=0 2024-11-24T09:43:24,816 DEBUG [RS:2;ba35ada4783f:46517 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295/ba35ada4783f%2C46517%2C1732441403295.1732441404802, exclude list is [], retry=0 2024-11-24T09:43:24,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:24,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:24,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:24,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:24,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:24,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:24,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:24,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:24,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:24,827 INFO [RS:0;ba35ada4783f:42075 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,42075,1732441403106/ba35ada4783f%2C42075%2C1732441403106.1732441404802 2024-11-24T09:43:24,827 INFO [RS:2;ba35ada4783f:46517 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295/ba35ada4783f%2C46517%2C1732441403295.1732441404802 2024-11-24T09:43:24,827 INFO [RS:1;ba35ada4783f:34867 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,34867,1732441403238/ba35ada4783f%2C34867%2C1732441403238.1732441404802 2024-11-24T09:43:24,827 DEBUG [RS:0;ba35ada4783f:42075 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:35185:35185)] 2024-11-24T09:43:24,827 DEBUG [RS:2;ba35ada4783f:46517 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:35185:35185)] 2024-11-24T09:43:24,827 DEBUG [RS:1;ba35ada4783f:34867 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:24,949 DEBUG [ba35ada4783f:39325 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T09:43:24,957 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(204): Hosts are {ba35ada4783f=0} racks are {/default-rack=0} 2024-11-24T09:43:24,963 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T09:43:24,963 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T09:43:24,963 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T09:43:24,963 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T09:43:24,963 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T09:43:24,963 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T09:43:24,964 INFO [ba35ada4783f:39325 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T09:43:24,964 INFO [ba35ada4783f:39325 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T09:43:24,964 INFO [ba35ada4783f:39325 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T09:43:24,964 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T09:43:24,971 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba35ada4783f,46517,1732441403295 2024-11-24T09:43:24,976 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba35ada4783f,46517,1732441403295, state=OPENING 2024-11-24T09:43:25,037 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T09:43:25,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:25,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:25,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:25,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:43:25,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:25,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:25,049 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:25,049 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:25,051 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T09:43:25,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba35ada4783f,46517,1732441403295}] 2024-11-24T09:43:25,230 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T09:43:25,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40675, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T09:43:25,244 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T09:43:25,244 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:25,245 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T09:43:25,248 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C46517%2C1732441403295.meta, suffix=.meta, logDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295, archiveDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/oldWALs, maxLogs=32 2024-11-24T09:43:25,263 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta, exclude list is [], retry=0 2024-11-24T09:43:25,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:25,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:25,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:25,270 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta 2024-11-24T09:43:25,270 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591)] 2024-11-24T09:43:25,271 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:25,272 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T09:43:25,274 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T09:43:25,279 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T09:43:25,282 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T09:43:25,283 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:25,283 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T09:43:25,283 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T09:43:25,286 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T09:43:25,287 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T09:43:25,288 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:25,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:25,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T09:43:25,290 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T09:43:25,290 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:25,291 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:25,291 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T09:43:25,292 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T09:43:25,292 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:25,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:25,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T09:43:25,294 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T09:43:25,294 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:25,295 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:25,295 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T09:43:25,296 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740 2024-11-24T09:43:25,299 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740 2024-11-24T09:43:25,301 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T09:43:25,301 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T09:43:25,302 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T09:43:25,305 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T09:43:25,306 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73239144, jitterRate=0.09134829044342041}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T09:43:25,307 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T09:43:25,308 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732441405284Writing region info on filesystem at 1732441405284Initializing all the Stores at 1732441405286 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441405286Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441405286Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441405286Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441405286Cleaning up temporary data from old regions at 1732441405301 (+15 ms)Running coprocessor post-open hooks at 1732441405307 (+6 ms)Region opened successfully at 1732441405308 (+1 ms) 2024-11-24T09:43:25,314 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732441405223 2024-11-24T09:43:25,324 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T09:43:25,325 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T09:43:25,327 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ba35ada4783f,46517,1732441403295 2024-11-24T09:43:25,329 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba35ada4783f,46517,1732441403295, state=OPEN 2024-11-24T09:43:25,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:43:25,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:43:25,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:43:25,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:43:25,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:25,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:25,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:25,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:25,362 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba35ada4783f,46517,1732441403295 2024-11-24T09:43:25,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T09:43:25,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba35ada4783f,46517,1732441403295 in 308 msec 2024-11-24T09:43:25,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T09:43:25,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 846 msec 2024-11-24T09:43:25,386 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T09:43:25,386 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T09:43:25,405 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:43:25,407 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1] 2024-11-24T09:43:25,431 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T09:43:25,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50305, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T09:43:25,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1570 sec 2024-11-24T09:43:25,452 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732441405452, completionTime=-1 2024-11-24T09:43:25,455 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T09:43:25,455 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T09:43:25,479 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T09:43:25,479 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732441465479 2024-11-24T09:43:25,479 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732441525479 2024-11-24T09:43:25,479 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-24T09:43:25,480 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T09:43:25,488 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,39325,1732441402377-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:25,488 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,39325,1732441402377-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:25,488 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,39325,1732441402377-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:25,489 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ba35ada4783f:39325, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:25,490 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:25,490 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:25,495 DEBUG [master/ba35ada4783f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T09:43:25,515 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.110sec 2024-11-24T09:43:25,517 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T09:43:25,518 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T09:43:25,519 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T09:43:25,519 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T09:43:25,519 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T09:43:25,520 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,39325,1732441402377-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T09:43:25,521 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,39325,1732441402377-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T09:43:25,525 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T09:43:25,526 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T09:43:25,527 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,39325,1732441402377-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:25,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ddf6023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T09:43:25,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ba35ada4783f,39325,-1 for getting cluster id 2024-11-24T09:43:25,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T09:43:25,562 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '574c19c5-3bf5-487a-a197-015a772295c0' 2024-11-24T09:43:25,564 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T09:43:25,564 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "574c19c5-3bf5-487a-a197-015a772295c0" 2024-11-24T09:43:25,566 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4794f32c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T09:43:25,566 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ba35ada4783f,39325,-1] 2024-11-24T09:43:25,569 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T09:43:25,571 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:43:25,572 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T09:43:25,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65c83e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T09:43:25,575 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:43:25,581 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1] 2024-11-24T09:43:25,582 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T09:43:25,584 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T09:43:25,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ba35ada4783f,39325,1732441402377 2024-11-24T09:43:25,602 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:36549/hbase 2024-11-24T09:43:25,616 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=360, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=345, ProcessCount=11, AvailableMemoryMB=8113 2024-11-24T09:43:25,634 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:25,638 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:25,640 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:25,644 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57636708, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/hregion-57636708, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:25,659 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57636708/hregion-57636708.1732441405646, exclude list is [], retry=0 2024-11-24T09:43:25,663 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:25,663 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:25,664 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:25,667 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57636708/hregion-57636708.1732441405646 2024-11-24T09:43:25,667 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591)] 2024-11-24T09:43:25,668 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 19288186771b3a6d41b9f25e9a86b272, NAME => 'testReplayEditsWrittenIntoWAL,,1732441405636.19288186771b3a6d41b9f25e9a86b272.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36549/hbase 2024-11-24T09:43:25,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741838_1014 (size=64) 2024-11-24T09:43:25,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741838_1014 (size=64) 2024-11-24T09:43:25,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741838_1014 (size=64) 2024-11-24T09:43:25,682 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732441405636.19288186771b3a6d41b9f25e9a86b272.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:25,685 INFO [StoreOpener-19288186771b3a6d41b9f25e9a86b272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,687 INFO [StoreOpener-19288186771b3a6d41b9f25e9a86b272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19288186771b3a6d41b9f25e9a86b272 columnFamilyName a 2024-11-24T09:43:25,688 DEBUG [StoreOpener-19288186771b3a6d41b9f25e9a86b272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:25,689 INFO [StoreOpener-19288186771b3a6d41b9f25e9a86b272-1 {}] regionserver.HStore(327): Store=19288186771b3a6d41b9f25e9a86b272/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:25,689 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,691 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,692 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,693 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,693 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,696 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,700 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:25,701 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 19288186771b3a6d41b9f25e9a86b272; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65005634, jitterRate=-0.031340569257736206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:43:25,703 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 19288186771b3a6d41b9f25e9a86b272: Writing region info on filesystem at 1732441405683Initializing all the Stores at 1732441405684 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441405684Cleaning up temporary data from old regions at 1732441405693 (+9 ms)Region opened successfully at 1732441405703 (+10 ms) 2024-11-24T09:43:25,703 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 19288186771b3a6d41b9f25e9a86b272, disabling compactions & flushes 2024-11-24T09:43:25,704 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732441405636.19288186771b3a6d41b9f25e9a86b272. 2024-11-24T09:43:25,704 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732441405636.19288186771b3a6d41b9f25e9a86b272. 2024-11-24T09:43:25,704 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732441405636.19288186771b3a6d41b9f25e9a86b272. after waiting 0 ms 2024-11-24T09:43:25,704 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732441405636.19288186771b3a6d41b9f25e9a86b272. 2024-11-24T09:43:25,704 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732441405636.19288186771b3a6d41b9f25e9a86b272. 2024-11-24T09:43:25,704 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 19288186771b3a6d41b9f25e9a86b272: Waiting for close lock at 1732441405703Disabling compacts and flushes for region at 1732441405703Disabling writes for close at 1732441405704 (+1 ms)Writing region close event to WAL at 1732441405704Closed at 1732441405704 2024-11-24T09:43:25,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741837_1013 (size=95) 2024-11-24T09:43:25,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741837_1013 (size=95) 2024-11-24T09:43:25,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741837_1013 (size=95) 2024-11-24T09:43:25,717 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:25,717 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57636708:(num 1732441405646) 2024-11-24T09:43:25,719 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T09:43:25,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741839_1015 (size=320) 2024-11-24T09:43:25,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741839_1015 (size=320) 2024-11-24T09:43:25,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741839_1015 (size=320) 2024-11-24T09:43:25,741 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T09:43:25,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741840_1016 (size=253) 2024-11-24T09:43:25,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741840_1016 (size=253) 2024-11-24T09:43:25,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741840_1016 (size=253) 2024-11-24T09:43:25,779 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-1, size=320 (320bytes) 2024-11-24T09:43:25,779 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T09:43:25,779 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T09:43:25,780 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-1 2024-11-24T09:43:25,785 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-1 after 3ms 2024-11-24T09:43:25,791 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:25,793 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-1 took 16ms 2024-11-24T09:43:25,803 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-1 so closing down 2024-11-24T09:43:25,803 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:25,806 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T09:43:25,808 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T09:43:25,809 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:25,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741841_1017 (size=320) 2024-11-24T09:43:25,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741841_1017 (size=320) 2024-11-24T09:43:25,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741841_1017 (size=320) 2024-11-24T09:43:25,822 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T09:43:25,825 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000002 2024-11-24T09:43:25,830 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-24T09:43:25,831 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-1, journal: Splitting hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-1, size=320 (320bytes) at 1732441405779Finishing writing output for hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-1 so closing down at 1732441405803 (+24 ms)Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000001-wal-1.temp at 1732441405808 (+5 ms)3 split writer threads finished at 1732441405810 (+2 ms)Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732441405822 (+12 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000002 at 1732441405825 (+3 ms)Processed 2 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732441405830 (+5 ms) 2024-11-24T09:43:25,848 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-2, size=253 (253bytes) 2024-11-24T09:43:25,848 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-2 2024-11-24T09:43:25,849 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-2 after 1ms 2024-11-24T09:43:25,853 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:25,854 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-2 took 7ms 2024-11-24T09:43:25,857 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-2 so closing down 2024-11-24T09:43:25,857 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:25,860 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T09:43:25,861 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T09:43:25,862 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:25,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741842_1018 (size=253) 2024-11-24T09:43:25,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741842_1018 (size=253) 2024-11-24T09:43:25,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741842_1018 (size=253) 2024-11-24T09:43:25,873 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T09:43:25,878 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:25,881 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-11-24T09:43:25,884 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-24T09:43:25,884 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-2, journal: Splitting hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-2, size=253 (253bytes) at 1732441405848Finishing writing output for hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-2 so closing down at 1732441405857 (+9 ms)Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000002-wal-2.temp at 1732441405861 (+4 ms)3 split writer threads finished at 1732441405862 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732441405873 (+11 ms)Processed 1 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732441405884 (+11 ms) 2024-11-24T09:43:25,884 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:25,887 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:25,906 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal.1732441405888, exclude list is [], retry=0 2024-11-24T09:43:25,910 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:25,911 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:25,911 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:25,915 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal.1732441405888 2024-11-24T09:43:25,916 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:35185:35185)] 2024-11-24T09:43:25,916 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 19288186771b3a6d41b9f25e9a86b272, NAME => 'testReplayEditsWrittenIntoWAL,,1732441405636.19288186771b3a6d41b9f25e9a86b272.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:25,916 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732441405636.19288186771b3a6d41b9f25e9a86b272.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:25,916 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,916 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,918 INFO [StoreOpener-19288186771b3a6d41b9f25e9a86b272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,920 INFO [StoreOpener-19288186771b3a6d41b9f25e9a86b272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19288186771b3a6d41b9f25e9a86b272 columnFamilyName a 2024-11-24T09:43:25,920 DEBUG [StoreOpener-19288186771b3a6d41b9f25e9a86b272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:25,921 INFO [StoreOpener-19288186771b3a6d41b9f25e9a86b272-1 {}] regionserver.HStore(327): Store=19288186771b3a6d41b9f25e9a86b272/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:25,921 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,922 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,925 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:25,926 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000002 2024-11-24T09:43:25,930 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:25,936 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000002 2024-11-24T09:43:25,939 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 19288186771b3a6d41b9f25e9a86b272 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T09:43:26,016 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/.tmp/a/d7bc03597ee14302a54e639cee0cca1a is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732441405717/Put/seqid=0 2024-11-24T09:43:26,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741844_1020 (size=5170) 2024-11-24T09:43:26,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741844_1020 (size=5170) 2024-11-24T09:43:26,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741844_1020 (size=5170) 2024-11-24T09:43:26,032 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/.tmp/a/d7bc03597ee14302a54e639cee0cca1a 2024-11-24T09:43:26,089 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/.tmp/a/d7bc03597ee14302a54e639cee0cca1a as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/a/d7bc03597ee14302a54e639cee0cca1a 2024-11-24T09:43:26,099 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/a/d7bc03597ee14302a54e639cee0cca1a, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T09:43:26,104 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 19288186771b3a6d41b9f25e9a86b272 in 164ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T09:43:26,106 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/0000000000000000002 2024-11-24T09:43:26,107 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:26,107 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:26,110 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 19288186771b3a6d41b9f25e9a86b272 2024-11-24T09:43:26,113 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/19288186771b3a6d41b9f25e9a86b272/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T09:43:26,115 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 19288186771b3a6d41b9f25e9a86b272; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74011424, jitterRate=0.10285615921020508}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:43:26,115 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 19288186771b3a6d41b9f25e9a86b272: Writing region info on filesystem at 1732441405916Initializing all the Stores at 1732441405918 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441405918Obtaining lock to block concurrent updates at 1732441405939 (+21 ms)Preparing flush snapshotting stores in 19288186771b3a6d41b9f25e9a86b272 at 1732441405939Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732441405636.19288186771b3a6d41b9f25e9a86b272., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732441405942 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732441405636.19288186771b3a6d41b9f25e9a86b272. at 1732441405942Flushing 19288186771b3a6d41b9f25e9a86b272/a: creating writer at 1732441405943 (+1 ms)Flushing 19288186771b3a6d41b9f25e9a86b272/a: appending metadata at 1732441405979 (+36 ms)Flushing 19288186771b3a6d41b9f25e9a86b272/a: closing flushed file at 1732441405982 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@668b3090: reopening flushed file at 1732441406088 (+106 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 19288186771b3a6d41b9f25e9a86b272 in 164ms, sequenceid=2, compaction requested=false; wal=null at 1732441406104 (+16 ms)Cleaning up temporary data from old regions at 1732441406107 (+3 ms)Region opened successfully at 1732441406115 (+8 ms) 2024-11-24T09:43:26,138 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=372 (was 360) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:41756 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:44760 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:36549/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:44620 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:35970 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:35954 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:41904 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=345 (was 345), ProcessCount=11 (was 11), AvailableMemoryMB=8028 (was 8113) 2024-11-24T09:43:26,148 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=372, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=345, ProcessCount=11, AvailableMemoryMB=8027 2024-11-24T09:43:26,165 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:26,167 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:26,168 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:26,172 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-35495573, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/hregion-35495573, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:26,187 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-35495573/hregion-35495573.1732441406172, exclude list is [], retry=0 2024-11-24T09:43:26,191 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:26,191 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:26,191 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:26,194 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-35495573/hregion-35495573.1732441406172 2024-11-24T09:43:26,195 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:35185:35185)] 2024-11-24T09:43:26,195 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2710bc0c8f75358a7ea5e4cab0b06118, NAME => 'testReplayEditsWrittenIntoWAL,,1732441406165.2710bc0c8f75358a7ea5e4cab0b06118.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36549/hbase 2024-11-24T09:43:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741846_1022 (size=64) 2024-11-24T09:43:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741846_1022 (size=64) 2024-11-24T09:43:26,208 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732441406165.2710bc0c8f75358a7ea5e4cab0b06118.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:26,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741846_1022 (size=64) 2024-11-24T09:43:26,210 INFO [StoreOpener-2710bc0c8f75358a7ea5e4cab0b06118-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,212 INFO [StoreOpener-2710bc0c8f75358a7ea5e4cab0b06118-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2710bc0c8f75358a7ea5e4cab0b06118 columnFamilyName a 2024-11-24T09:43:26,212 DEBUG [StoreOpener-2710bc0c8f75358a7ea5e4cab0b06118-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:26,213 INFO [StoreOpener-2710bc0c8f75358a7ea5e4cab0b06118-1 {}] regionserver.HStore(327): Store=2710bc0c8f75358a7ea5e4cab0b06118/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:26,213 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,213 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,214 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,214 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,215 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,216 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,219 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:26,220 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2710bc0c8f75358a7ea5e4cab0b06118; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67449093, jitterRate=0.005069807171821594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:43:26,220 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2710bc0c8f75358a7ea5e4cab0b06118: Writing region info on filesystem at 1732441406208Initializing all the Stores at 1732441406209 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441406209Cleaning up temporary data from old regions at 1732441406215 (+6 ms)Region opened successfully at 1732441406220 (+5 ms) 2024-11-24T09:43:26,220 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2710bc0c8f75358a7ea5e4cab0b06118, disabling compactions & flushes 2024-11-24T09:43:26,220 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732441406165.2710bc0c8f75358a7ea5e4cab0b06118. 2024-11-24T09:43:26,221 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732441406165.2710bc0c8f75358a7ea5e4cab0b06118. 2024-11-24T09:43:26,221 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732441406165.2710bc0c8f75358a7ea5e4cab0b06118. after waiting 0 ms 2024-11-24T09:43:26,221 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732441406165.2710bc0c8f75358a7ea5e4cab0b06118. 2024-11-24T09:43:26,221 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732441406165.2710bc0c8f75358a7ea5e4cab0b06118. 2024-11-24T09:43:26,221 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2710bc0c8f75358a7ea5e4cab0b06118: Waiting for close lock at 1732441406220Disabling compacts and flushes for region at 1732441406220Disabling writes for close at 1732441406221 (+1 ms)Writing region close event to WAL at 1732441406221Closed at 1732441406221 2024-11-24T09:43:26,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741845_1021 (size=95) 2024-11-24T09:43:26,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741845_1021 (size=95) 2024-11-24T09:43:26,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741845_1021 (size=95) 2024-11-24T09:43:26,227 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:26,227 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-35495573:(num 1732441406172) 2024-11-24T09:43:26,228 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T09:43:26,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741847_1023 (size=320) 2024-11-24T09:43:26,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741847_1023 (size=320) 2024-11-24T09:43:26,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741847_1023 (size=320) 2024-11-24T09:43:26,240 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T09:43:26,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741848_1024 (size=253) 2024-11-24T09:43:26,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741848_1024 (size=253) 2024-11-24T09:43:26,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741848_1024 (size=253) 2024-11-24T09:43:26,267 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-2, size=253 (253bytes) 2024-11-24T09:43:26,267 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-2 2024-11-24T09:43:26,268 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-2 after 1ms 2024-11-24T09:43:26,271 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:26,272 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-2 took 5ms 2024-11-24T09:43:26,274 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-2 so closing down 2024-11-24T09:43:26,274 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:26,277 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T09:43:26,278 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T09:43:26,279 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:26,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741849_1025 (size=253) 2024-11-24T09:43:26,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741849_1025 (size=253) 2024-11-24T09:43:26,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741849_1025 (size=253) 2024-11-24T09:43:26,287 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T09:43:26,289 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002 2024-11-24T09:43:26,289 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-24T09:43:26,289 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-2, journal: Splitting hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-2, size=253 (253bytes) at 1732441406267Finishing writing output for hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-2 so closing down at 1732441406274 (+7 ms)Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002-wal-2.temp at 1732441406278 (+4 ms)3 split writer threads finished at 1732441406279 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732441406287 (+8 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002 at 1732441406289 (+2 ms)Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732441406289 2024-11-24T09:43:26,306 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-1, size=320 (320bytes) 2024-11-24T09:43:26,306 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-1 2024-11-24T09:43:26,306 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-1 after 0ms 2024-11-24T09:43:26,310 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:26,310 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-1 took 5ms 2024-11-24T09:43:26,313 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-1 so closing down 2024-11-24T09:43:26,313 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:26,315 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T09:43:26,317 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T09:43:26,317 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:26,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741850_1026 (size=320) 2024-11-24T09:43:26,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741850_1026 (size=320) 2024-11-24T09:43:26,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741850_1026 (size=320) 2024-11-24T09:43:26,326 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T09:43:26,330 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:26,332 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002, length=253 2024-11-24T09:43:26,334 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002 2024-11-24T09:43:26,334 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-24T09:43:26,335 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-1, journal: Splitting hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-1, size=320 (320bytes) at 1732441406306Finishing writing output for hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-1 so closing down at 1732441406313 (+7 ms)Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000001-wal-1.temp at 1732441406317 (+4 ms)3 split writer threads finished at 1732441406317Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732441406326 (+9 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002 at 1732441406334 (+8 ms)Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732441406335 (+1 ms) 2024-11-24T09:43:26,335 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:26,337 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:26,349 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal.1732441406337, exclude list is [], retry=0 2024-11-24T09:43:26,353 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:26,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:26,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:26,356 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal.1732441406337 2024-11-24T09:43:26,357 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591)] 2024-11-24T09:43:26,357 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2710bc0c8f75358a7ea5e4cab0b06118, NAME => 'testReplayEditsWrittenIntoWAL,,1732441406165.2710bc0c8f75358a7ea5e4cab0b06118.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:26,357 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732441406165.2710bc0c8f75358a7ea5e4cab0b06118.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:26,357 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,357 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,359 INFO [StoreOpener-2710bc0c8f75358a7ea5e4cab0b06118-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,361 INFO [StoreOpener-2710bc0c8f75358a7ea5e4cab0b06118-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2710bc0c8f75358a7ea5e4cab0b06118 columnFamilyName a 2024-11-24T09:43:26,361 DEBUG [StoreOpener-2710bc0c8f75358a7ea5e4cab0b06118-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:26,362 INFO [StoreOpener-2710bc0c8f75358a7ea5e4cab0b06118-1 {}] regionserver.HStore(327): Store=2710bc0c8f75358a7ea5e4cab0b06118/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:26,362 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,363 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,365 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,366 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002 2024-11-24T09:43:26,369 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:26,371 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002 2024-11-24T09:43:26,371 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2710bc0c8f75358a7ea5e4cab0b06118 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T09:43:26,387 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/.tmp/a/fe8d22f47ce44b108f0fe1f6b59f4fb5 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732441406227/Put/seqid=0 2024-11-24T09:43:26,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741852_1028 (size=5170) 2024-11-24T09:43:26,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741852_1028 (size=5170) 2024-11-24T09:43:26,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741852_1028 (size=5170) 2024-11-24T09:43:26,396 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/.tmp/a/fe8d22f47ce44b108f0fe1f6b59f4fb5 2024-11-24T09:43:26,407 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/.tmp/a/fe8d22f47ce44b108f0fe1f6b59f4fb5 as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/a/fe8d22f47ce44b108f0fe1f6b59f4fb5 2024-11-24T09:43:26,416 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/a/fe8d22f47ce44b108f0fe1f6b59f4fb5, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T09:43:26,416 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 2710bc0c8f75358a7ea5e4cab0b06118 in 45ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T09:43:26,417 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/0000000000000000002 2024-11-24T09:43:26,417 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,417 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,420 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2710bc0c8f75358a7ea5e4cab0b06118 2024-11-24T09:43:26,423 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/2710bc0c8f75358a7ea5e4cab0b06118/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T09:43:26,424 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2710bc0c8f75358a7ea5e4cab0b06118; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71873080, jitterRate=0.0709923505783081}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:43:26,425 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2710bc0c8f75358a7ea5e4cab0b06118: Writing region info on filesystem at 1732441406357Initializing all the Stores at 1732441406359 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441406359Obtaining lock to block concurrent updates at 1732441406371 (+12 ms)Preparing flush snapshotting stores in 2710bc0c8f75358a7ea5e4cab0b06118 at 1732441406371Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732441406165.2710bc0c8f75358a7ea5e4cab0b06118., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732441406371Flushing stores of testReplayEditsWrittenIntoWAL,,1732441406165.2710bc0c8f75358a7ea5e4cab0b06118. at 1732441406371Flushing 2710bc0c8f75358a7ea5e4cab0b06118/a: creating writer at 1732441406371Flushing 2710bc0c8f75358a7ea5e4cab0b06118/a: appending metadata at 1732441406386 (+15 ms)Flushing 2710bc0c8f75358a7ea5e4cab0b06118/a: closing flushed file at 1732441406386Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d616c34: reopening flushed file at 1732441406406 (+20 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 2710bc0c8f75358a7ea5e4cab0b06118 in 45ms, sequenceid=2, compaction requested=false; wal=null at 1732441406416 (+10 ms)Cleaning up temporary data from old regions at 1732441406417 (+1 ms)Region opened successfully at 1732441406424 (+7 ms) 2024-11-24T09:43:26,443 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=382 (was 372) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:41982 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:41756 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:44828 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:36030 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:44620 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:35954 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=345 (was 345), ProcessCount=11 (was 11), AvailableMemoryMB=8018 (was 8027) 2024-11-24T09:43:26,452 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=382, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=345, ProcessCount=11, AvailableMemoryMB=8018 2024-11-24T09:43:26,467 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:26,469 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:26,470 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:26,473 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-67012961, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/hregion-67012961, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:26,491 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-67012961/hregion-67012961.1732441406474, exclude list is [], retry=0 2024-11-24T09:43:26,495 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:26,495 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:26,495 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:26,497 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-67012961/hregion-67012961.1732441406474 2024-11-24T09:43:26,498 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:26,498 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9a396eb4c95958545baafca55727ea43, NAME => 'testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36549/hbase 2024-11-24T09:43:26,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741854_1030 (size=64) 2024-11-24T09:43:26,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741854_1030 (size=64) 2024-11-24T09:43:26,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741854_1030 (size=64) 2024-11-24T09:43:26,509 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:26,510 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:26,512 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a396eb4c95958545baafca55727ea43 columnFamilyName a 2024-11-24T09:43:26,512 DEBUG [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:26,513 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] regionserver.HStore(327): Store=9a396eb4c95958545baafca55727ea43/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:26,513 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:26,515 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a396eb4c95958545baafca55727ea43 columnFamilyName b 2024-11-24T09:43:26,515 DEBUG [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:26,515 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] regionserver.HStore(327): Store=9a396eb4c95958545baafca55727ea43/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:26,515 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:26,517 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a396eb4c95958545baafca55727ea43 columnFamilyName c 2024-11-24T09:43:26,517 DEBUG [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:26,518 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] regionserver.HStore(327): Store=9a396eb4c95958545baafca55727ea43/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:26,519 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:26,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:26,520 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:26,521 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:26,521 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:26,522 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:26,524 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:26,526 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:26,527 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9a396eb4c95958545baafca55727ea43; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73574711, jitterRate=0.09634862840175629}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:26,527 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9a396eb4c95958545baafca55727ea43: Writing region info on filesystem at 1732441406509Initializing all the Stores at 1732441406510 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441406510Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441406510Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441406510Cleaning up temporary data from old regions at 1732441406521 (+11 ms)Region opened successfully at 1732441406527 (+6 ms) 2024-11-24T09:43:26,527 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9a396eb4c95958545baafca55727ea43, disabling compactions & flushes 2024-11-24T09:43:26,527 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43. 2024-11-24T09:43:26,527 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43. 2024-11-24T09:43:26,527 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43. after waiting 0 ms 2024-11-24T09:43:26,527 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43. 2024-11-24T09:43:26,528 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43. 2024-11-24T09:43:26,528 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9a396eb4c95958545baafca55727ea43: Waiting for close lock at 1732441406527Disabling compacts and flushes for region at 1732441406527Disabling writes for close at 1732441406527Writing region close event to WAL at 1732441406528 (+1 ms)Closed at 1732441406528 2024-11-24T09:43:26,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741853_1029 (size=95) 2024-11-24T09:43:26,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741853_1029 (size=95) 2024-11-24T09:43:26,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741853_1029 (size=95) 2024-11-24T09:43:26,535 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:26,535 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-67012961:(num 1732441406474) 2024-11-24T09:43:26,535 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:26,537 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:26,551 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538, exclude list is [], retry=0 2024-11-24T09:43:26,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:26,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:26,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:26,557 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538 2024-11-24T09:43:26,558 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:26,829 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538, size=0 (0bytes) 2024-11-24T09:43:26,829 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538 might be still open, length is 0 2024-11-24T09:43:26,829 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538 2024-11-24T09:43:26,830 WARN [IPC Server handler 2 on default port 36549 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-24T09:43:26,830 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538 after 1ms 2024-11-24T09:43:27,533 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:44868 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37953:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44868 dst: /127.0.0.1:37953 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37953 remote=/127.0.0.1:44868]. Total timeout mills is 60000, 59256 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:27,534 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:36072 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37499:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36072 dst: /127.0.0.1:37499 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:27,535 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:42014 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42014 dst: /127.0.0.1:33653 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:27,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741855_1032 (size=263633) 2024-11-24T09:43:27,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741855_1032 (size=263633) 2024-11-24T09:43:27,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741855_1032 (size=263633) 2024-11-24T09:43:30,795 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T09:43:30,831 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538 after 4002ms 2024-11-24T09:43:30,835 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:30,837 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538 took 4009ms 2024-11-24T09:43:30,841 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732441406538.temp 2024-11-24T09:43:30,843 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000000001-wal.1732441406538.temp 2024-11-24T09:43:30,857 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T09:43:30,982 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538; continuing. 2024-11-24T09:43:30,982 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538 so closing down 2024-11-24T09:43:30,982 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:30,982 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:30,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741856_1033 (size=263641) 2024-11-24T09:43:30,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741856_1033 (size=263641) 2024-11-24T09:43:30,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741856_1033 (size=263641) 2024-11-24T09:43:30,986 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000000001-wal.1732441406538.temp (wrote 3002 edits, skipped 0 edits in 76 ms) 2024-11-24T09:43:30,988 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000000001-wal.1732441406538.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000003002 2024-11-24T09:43:30,988 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 151 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T09:43:30,989 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538, journal: Splitting hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538, size=0 (0bytes) at 1732441406829Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000000001-wal.1732441406538.temp at 1732441410843 (+4014 ms)Split 1024 edits, skipped 0 edits. at 1732441410920 (+77 ms)Split 2048 edits, skipped 0 edits. at 1732441410953 (+33 ms)Finishing writing output for hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538 so closing down at 1732441410982 (+29 ms)3 split writer threads finished at 1732441410982Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000000001-wal.1732441406538.temp (wrote 3002 edits, skipped 0 edits in 76 ms) at 1732441410986 (+4 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000000001-wal.1732441406538.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000003002 at 1732441410988 (+2 ms)Processed 3002 edits across 1 Regions in 151 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538, size=0, length=0, corrupted=false, cancelled=false at 1732441410989 (+1 ms) 2024-11-24T09:43:30,991 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538 to hdfs://localhost:36549/hbase/oldWALs/wal.1732441406538 2024-11-24T09:43:30,993 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000003002 2024-11-24T09:43:30,994 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:30,997 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:31,015 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441410997, exclude list is [], retry=0 2024-11-24T09:43:31,019 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:31,019 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:31,019 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:31,022 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441410997 2024-11-24T09:43:31,023 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591)] 2024-11-24T09:43:31,023 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:31,026 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:31,027 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a396eb4c95958545baafca55727ea43 columnFamilyName a 2024-11-24T09:43:31,028 DEBUG [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:31,028 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] regionserver.HStore(327): Store=9a396eb4c95958545baafca55727ea43/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:31,028 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:31,030 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a396eb4c95958545baafca55727ea43 columnFamilyName b 2024-11-24T09:43:31,030 DEBUG [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:31,030 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] regionserver.HStore(327): Store=9a396eb4c95958545baafca55727ea43/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:31,031 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:31,032 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a396eb4c95958545baafca55727ea43 columnFamilyName c 2024-11-24T09:43:31,032 DEBUG [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:31,033 INFO [StoreOpener-9a396eb4c95958545baafca55727ea43-1 {}] regionserver.HStore(327): Store=9a396eb4c95958545baafca55727ea43/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:31,033 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:31,034 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:31,036 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:31,038 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000003002 2024-11-24T09:43:31,041 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:31,095 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T09:43:31,466 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9a396eb4c95958545baafca55727ea43 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T09:43:31,510 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/a/306836862a744bcf9263302bd3f2485c is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732441406569/Put/seqid=0 2024-11-24T09:43:31,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741858_1035 (size=50463) 2024-11-24T09:43:31,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741858_1035 (size=50463) 2024-11-24T09:43:31,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741858_1035 (size=50463) 2024-11-24T09:43:31,522 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/a/306836862a744bcf9263302bd3f2485c 2024-11-24T09:43:31,530 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/a/306836862a744bcf9263302bd3f2485c as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/a/306836862a744bcf9263302bd3f2485c 2024-11-24T09:43:31,537 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/a/306836862a744bcf9263302bd3f2485c, entries=754, sequenceid=754, filesize=49.3 K 2024-11-24T09:43:31,537 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 9a396eb4c95958545baafca55727ea43 in 72ms, sequenceid=754, compaction requested=false; wal=null 2024-11-24T09:43:31,565 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T09:43:31,566 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9a396eb4c95958545baafca55727ea43 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T09:43:31,574 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/a/4351ce365566474787e384f9efea8334 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732441406627/Put/seqid=0 2024-11-24T09:43:31,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741859_1036 (size=20072) 2024-11-24T09:43:31,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741859_1036 (size=20072) 2024-11-24T09:43:31,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741859_1036 (size=20072) 2024-11-24T09:43:31,587 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/a/4351ce365566474787e384f9efea8334 2024-11-24T09:43:31,613 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/b/b74f2d5d932d42468d85d375756686c5 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732441406654/Put/seqid=0 2024-11-24T09:43:31,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741860_1037 (size=35835) 2024-11-24T09:43:31,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741860_1037 (size=35835) 2024-11-24T09:43:31,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741860_1037 (size=35835) 2024-11-24T09:43:31,621 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/b/b74f2d5d932d42468d85d375756686c5 2024-11-24T09:43:31,628 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/a/4351ce365566474787e384f9efea8334 as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/a/4351ce365566474787e384f9efea8334 2024-11-24T09:43:31,635 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/a/4351ce365566474787e384f9efea8334, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-24T09:43:31,637 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/b/b74f2d5d932d42468d85d375756686c5 as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/b/b74f2d5d932d42468d85d375756686c5 2024-11-24T09:43:31,643 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/b/b74f2d5d932d42468d85d375756686c5, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-24T09:43:31,643 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 9a396eb4c95958545baafca55727ea43 in 78ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-24T09:43:31,666 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T09:43:31,666 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9a396eb4c95958545baafca55727ea43 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T09:43:31,675 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/b/81841674f6a446aa8b7a037260cc7cfa is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732441406682/Put/seqid=0 2024-11-24T09:43:31,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741861_1038 (size=35082) 2024-11-24T09:43:31,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741861_1038 (size=35082) 2024-11-24T09:43:31,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741861_1038 (size=35082) 2024-11-24T09:43:31,685 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/b/81841674f6a446aa8b7a037260cc7cfa 2024-11-24T09:43:31,710 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/c/a176a18805364266a6ce73156175524f is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732441406726/Put/seqid=0 2024-11-24T09:43:31,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741862_1039 (size=20825) 2024-11-24T09:43:31,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741862_1039 (size=20825) 2024-11-24T09:43:31,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741862_1039 (size=20825) 2024-11-24T09:43:31,719 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/c/a176a18805364266a6ce73156175524f 2024-11-24T09:43:31,728 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/b/81841674f6a446aa8b7a037260cc7cfa as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/b/81841674f6a446aa8b7a037260cc7cfa 2024-11-24T09:43:31,736 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/b/81841674f6a446aa8b7a037260cc7cfa, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-24T09:43:31,738 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/c/a176a18805364266a6ce73156175524f as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/c/a176a18805364266a6ce73156175524f 2024-11-24T09:43:31,746 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/c/a176a18805364266a6ce73156175524f, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-24T09:43:31,746 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 9a396eb4c95958545baafca55727ea43 in 80ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-24T09:43:31,763 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732441406787/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43. 2024-11-24T09:43:31,766 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000003002 2024-11-24T09:43:31,767 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T09:43:31,767 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9a396eb4c95958545baafca55727ea43 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-24T09:43:31,777 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/c/670b7421f4c54a7ab3ed7bea182df77d is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732441406737/Put/seqid=0 2024-11-24T09:43:31,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741863_1040 (size=50301) 2024-11-24T09:43:31,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741863_1040 (size=50301) 2024-11-24T09:43:31,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741863_1040 (size=50301) 2024-11-24T09:43:31,788 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/c/670b7421f4c54a7ab3ed7bea182df77d 2024-11-24T09:43:31,793 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 670b7421f4c54a7ab3ed7bea182df77d 2024-11-24T09:43:31,795 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/.tmp/c/670b7421f4c54a7ab3ed7bea182df77d as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/c/670b7421f4c54a7ab3ed7bea182df77d 2024-11-24T09:43:31,801 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 670b7421f4c54a7ab3ed7bea182df77d 2024-11-24T09:43:31,802 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/c/670b7421f4c54a7ab3ed7bea182df77d, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-24T09:43:31,802 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 9a396eb4c95958545baafca55727ea43 in 35ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-24T09:43:31,803 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/0000000000000003002 2024-11-24T09:43:31,804 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:31,804 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:31,805 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T09:43:31,806 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9a396eb4c95958545baafca55727ea43 2024-11-24T09:43:31,809 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenIntoWAL/9a396eb4c95958545baafca55727ea43/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-24T09:43:31,811 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9a396eb4c95958545baafca55727ea43; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68458456, jitterRate=0.020110487937927246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T09:43:31,811 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9a396eb4c95958545baafca55727ea43: Writing region info on filesystem at 1732441411023Initializing all the Stores at 1732441411024 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441411024Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441411025 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441411025Cleaning up temporary data from old regions at 1732441411804 (+779 ms)Region opened successfully at 1732441411811 (+7 ms) 2024-11-24T09:43:31,878 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9a396eb4c95958545baafca55727ea43, disabling compactions & flushes 2024-11-24T09:43:31,878 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43. 2024-11-24T09:43:31,878 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43. 2024-11-24T09:43:31,878 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43. after waiting 0 ms 2024-11-24T09:43:31,878 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43. 2024-11-24T09:43:31,880 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732441406468.9a396eb4c95958545baafca55727ea43. 2024-11-24T09:43:31,880 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9a396eb4c95958545baafca55727ea43: Waiting for close lock at 1732441411878Disabling compacts and flushes for region at 1732441411878Disabling writes for close at 1732441411878Writing region close event to WAL at 1732441411880 (+2 ms)Closed at 1732441411880 2024-11-24T09:43:31,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741857_1034 (size=95) 2024-11-24T09:43:31,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741857_1034 (size=95) 2024-11-24T09:43:31,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741857_1034 (size=95) 2024-11-24T09:43:31,888 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:31,888 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732441410997) 2024-11-24T09:43:31,905 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=396 (was 382) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:41089 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:36549 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43857 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_422587792_22 at /127.0.0.1:42056 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41089 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43857 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:36549 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_422587792_22 at /127.0.0.1:44910 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_422587792_22 at /127.0.0.1:36134 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_422587792_22 at /127.0.0.1:44932 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 345), ProcessCount=11 (was 11), AvailableMemoryMB=7956 (was 8018) 2024-11-24T09:43:31,917 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=396, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=7954 2024-11-24T09:43:31,932 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:31,934 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:31,935 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:31,939 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-41985343, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/hregion-41985343, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:31,952 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-41985343/hregion-41985343.1732441411939, exclude list is [], retry=0 2024-11-24T09:43:31,956 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:31,956 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:31,957 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:31,959 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-41985343/hregion-41985343.1732441411939 2024-11-24T09:43:31,959 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:35185:35185)] 2024-11-24T09:43:31,959 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4ed028ae71179ed6d0052b206d002036, NAME => 'test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36549/hbase 2024-11-24T09:43:31,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741865_1042 (size=43) 2024-11-24T09:43:31,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741865_1042 (size=43) 2024-11-24T09:43:31,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741865_1042 (size=43) 2024-11-24T09:43:31,978 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:31,980 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:31,982 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed028ae71179ed6d0052b206d002036 columnFamilyName a 2024-11-24T09:43:31,982 DEBUG [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:31,983 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] regionserver.HStore(327): Store=4ed028ae71179ed6d0052b206d002036/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:31,983 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:31,985 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed028ae71179ed6d0052b206d002036 columnFamilyName b 2024-11-24T09:43:31,985 DEBUG [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:31,986 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] regionserver.HStore(327): Store=4ed028ae71179ed6d0052b206d002036/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:31,986 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:31,988 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed028ae71179ed6d0052b206d002036 columnFamilyName c 2024-11-24T09:43:31,988 DEBUG [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:31,989 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] regionserver.HStore(327): Store=4ed028ae71179ed6d0052b206d002036/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:31,989 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:31,990 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:31,991 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:31,992 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:31,992 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:31,993 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:31,994 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:31,997 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:31,998 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4ed028ae71179ed6d0052b206d002036; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61941537, jitterRate=-0.07699917256832123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:31,999 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4ed028ae71179ed6d0052b206d002036: Writing region info on filesystem at 1732441411978Initializing all the Stores at 1732441411980 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441411980Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441411980Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441411980Cleaning up temporary data from old regions at 1732441411992 (+12 ms)Region opened successfully at 1732441411999 (+7 ms) 2024-11-24T09:43:31,999 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4ed028ae71179ed6d0052b206d002036, disabling compactions & flushes 2024-11-24T09:43:31,999 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036. 2024-11-24T09:43:31,999 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036. 2024-11-24T09:43:31,999 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036. after waiting 0 ms 2024-11-24T09:43:31,999 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036. 2024-11-24T09:43:32,000 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036. 2024-11-24T09:43:32,000 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4ed028ae71179ed6d0052b206d002036: Waiting for close lock at 1732441411999Disabling compacts and flushes for region at 1732441411999Disabling writes for close at 1732441411999Writing region close event to WAL at 1732441412000 (+1 ms)Closed at 1732441412000 2024-11-24T09:43:32,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741864_1041 (size=95) 2024-11-24T09:43:32,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741864_1041 (size=95) 2024-11-24T09:43:32,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741864_1041 (size=95) 2024-11-24T09:43:32,006 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:32,006 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-41985343:(num 1732441411939) 2024-11-24T09:43:32,006 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:32,008 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:32,020 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008, exclude list is [], retry=0 2024-11-24T09:43:32,023 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:32,023 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:32,023 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:32,026 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008 2024-11-24T09:43:32,027 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:32,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741866_1043 (size=263359) 2024-11-24T09:43:32,188 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008 not finished, retry = 0 2024-11-24T09:43:32,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741866_1043 (size=263359) 2024-11-24T09:43:32,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741866_1043 (size=263359) 2024-11-24T09:43:32,311 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008, size=257.2 K (263359bytes) 2024-11-24T09:43:32,311 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008 2024-11-24T09:43:32,311 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008 after 0ms 2024-11-24T09:43:32,314 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:32,315 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008 took 5ms 2024-11-24T09:43:32,321 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732441412008.temp 2024-11-24T09:43:32,323 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000000001-wal.1732441412008.temp 2024-11-24T09:43:32,378 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008 so closing down 2024-11-24T09:43:32,378 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:32,378 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:32,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741867_1044 (size=263359) 2024-11-24T09:43:32,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741867_1044 (size=263359) 2024-11-24T09:43:32,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741867_1044 (size=263359) 2024-11-24T09:43:32,388 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000000001-wal.1732441412008.temp (wrote 3000 edits, skipped 0 edits in 36 ms) 2024-11-24T09:43:32,390 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000000001-wal.1732441412008.temp to hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003000 2024-11-24T09:43:32,390 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-11-24T09:43:32,390 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008, journal: Splitting hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008, size=257.2 K (263359bytes) at 1732441412311Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000000001-wal.1732441412008.temp at 1732441412323 (+12 ms)Split 1024 edits, skipped 0 edits. at 1732441412341 (+18 ms)Split 2048 edits, skipped 0 edits. at 1732441412360 (+19 ms)Finishing writing output for hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008 so closing down at 1732441412378 (+18 ms)3 split writer threads finished at 1732441412378Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000000001-wal.1732441412008.temp (wrote 3000 edits, skipped 0 edits in 36 ms) at 1732441412388 (+10 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000000001-wal.1732441412008.temp to hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003000 at 1732441412390 (+2 ms)Processed 3000 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1732441412390 2024-11-24T09:43:32,392 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412008 to hdfs://localhost:36549/hbase/oldWALs/wal.1732441412008 2024-11-24T09:43:32,393 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003000 2024-11-24T09:43:32,394 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:32,396 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:32,410 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396, exclude list is [], retry=0 2024-11-24T09:43:32,413 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:32,414 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:32,414 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:32,416 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396 2024-11-24T09:43:32,417 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:32,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741868_1045 (size=263486) 2024-11-24T09:43:32,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741868_1045 (size=263486) 2024-11-24T09:43:32,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741868_1045 (size=263486) 2024-11-24T09:43:32,595 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396, size=257.3 K (263486bytes) 2024-11-24T09:43:32,595 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396 2024-11-24T09:43:32,596 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396 after 1ms 2024-11-24T09:43:32,599 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:32,602 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396 took 7ms 2024-11-24T09:43:32,609 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732441412396.temp 2024-11-24T09:43:32,611 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003001-wal.1732441412396.temp 2024-11-24T09:43:32,660 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396 so closing down 2024-11-24T09:43:32,660 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:32,661 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:32,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741869_1046 (size=263486) 2024-11-24T09:43:32,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741869_1046 (size=263486) 2024-11-24T09:43:32,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741869_1046 (size=263486) 2024-11-24T09:43:32,665 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003001-wal.1732441412396.temp (wrote 3000 edits, skipped 0 edits in 35 ms) 2024-11-24T09:43:32,667 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003001-wal.1732441412396.temp to hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000006000 2024-11-24T09:43:32,667 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 64 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-11-24T09:43:32,667 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396, journal: Splitting hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396, size=257.3 K (263486bytes) at 1732441412595Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003001-wal.1732441412396.temp at 1732441412611 (+16 ms)Split 1024 edits, skipped 0 edits. at 1732441412623 (+12 ms)Split 2048 edits, skipped 0 edits. at 1732441412642 (+19 ms)Finishing writing output for hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396 so closing down at 1732441412660 (+18 ms)3 split writer threads finished at 1732441412661 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003001-wal.1732441412396.temp (wrote 3000 edits, skipped 0 edits in 35 ms) at 1732441412665 (+4 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003001-wal.1732441412396.temp to hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000006000 at 1732441412667 (+2 ms)Processed 3000 edits across 1 Regions in 64 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1732441412667 2024-11-24T09:43:32,670 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412396 to hdfs://localhost:36549/hbase/oldWALs/wal.1732441412396 2024-11-24T09:43:32,671 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000006000 2024-11-24T09:43:32,671 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:32,673 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/test2727-manual,16010,1732441411931, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:32,691 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412674, exclude list is [], retry=0 2024-11-24T09:43:32,695 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:32,695 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:32,696 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:32,698 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732441411931/wal.1732441412674 2024-11-24T09:43:32,699 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:35185:35185)] 2024-11-24T09:43:32,699 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4ed028ae71179ed6d0052b206d002036, NAME => 'test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:32,699 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:32,699 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:32,699 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:32,701 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:32,703 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed028ae71179ed6d0052b206d002036 columnFamilyName a 2024-11-24T09:43:32,703 DEBUG [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:32,704 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] regionserver.HStore(327): Store=4ed028ae71179ed6d0052b206d002036/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:32,704 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:32,705 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed028ae71179ed6d0052b206d002036 columnFamilyName b 2024-11-24T09:43:32,705 DEBUG [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:32,706 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] regionserver.HStore(327): Store=4ed028ae71179ed6d0052b206d002036/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:32,706 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:32,707 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed028ae71179ed6d0052b206d002036 columnFamilyName c 2024-11-24T09:43:32,707 DEBUG [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:32,708 INFO [StoreOpener-4ed028ae71179ed6d0052b206d002036-1 {}] regionserver.HStore(327): Store=4ed028ae71179ed6d0052b206d002036/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:32,708 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:32,709 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T09:43:32,709 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:32,709 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T09:43:32,712 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:32,712 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T09:43:32,712 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T09:43:32,712 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T09:43:32,712 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T09:43:32,712 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003000 2024-11-24T09:43:32,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-24T09:43:32,713 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-24T09:43:32,715 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:32,761 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003000 2024-11-24T09:43:32,762 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000006000 2024-11-24T09:43:32,765 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:32,811 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000006000 2024-11-24T09:43:32,811 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4ed028ae71179ed6d0052b206d002036 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-24T09:43:32,833 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/.tmp/a/8b32b7a82de740e9b34f74928a733498 is 41, key is test2727/a:100/1732441412423/Put/seqid=0 2024-11-24T09:43:32,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741871_1048 (size=84227) 2024-11-24T09:43:32,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741871_1048 (size=84227) 2024-11-24T09:43:32,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741871_1048 (size=84227) 2024-11-24T09:43:32,840 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/.tmp/a/8b32b7a82de740e9b34f74928a733498 2024-11-24T09:43:32,869 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/.tmp/b/30c12f8f38a146c6a2caa36eeb086082 is 41, key is test2727/b:100/1732441412475/Put/seqid=0 2024-11-24T09:43:32,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741872_1049 (size=84609) 2024-11-24T09:43:32,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741872_1049 (size=84609) 2024-11-24T09:43:32,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741872_1049 (size=84609) 2024-11-24T09:43:32,875 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/.tmp/b/30c12f8f38a146c6a2caa36eeb086082 2024-11-24T09:43:32,910 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/.tmp/c/1eb7f698a12b4dd3805903fa12382efb is 41, key is test2727/c:100/1732441412522/Put/seqid=0 2024-11-24T09:43:32,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741873_1050 (size=84609) 2024-11-24T09:43:32,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741873_1050 (size=84609) 2024-11-24T09:43:32,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741873_1050 (size=84609) 2024-11-24T09:43:32,917 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/.tmp/c/1eb7f698a12b4dd3805903fa12382efb 2024-11-24T09:43:32,925 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/.tmp/a/8b32b7a82de740e9b34f74928a733498 as hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/a/8b32b7a82de740e9b34f74928a733498 2024-11-24T09:43:32,933 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/a/8b32b7a82de740e9b34f74928a733498, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-24T09:43:32,934 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/.tmp/b/30c12f8f38a146c6a2caa36eeb086082 as hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/b/30c12f8f38a146c6a2caa36eeb086082 2024-11-24T09:43:32,942 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/b/30c12f8f38a146c6a2caa36eeb086082, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T09:43:32,944 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/.tmp/c/1eb7f698a12b4dd3805903fa12382efb as hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/c/1eb7f698a12b4dd3805903fa12382efb 2024-11-24T09:43:32,950 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/c/1eb7f698a12b4dd3805903fa12382efb, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T09:43:32,951 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 4ed028ae71179ed6d0052b206d002036 in 140ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-24T09:43:32,952 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000003000 2024-11-24T09:43:32,952 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/0000000000000006000 2024-11-24T09:43:32,954 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:32,954 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:32,954 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:32,956 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4ed028ae71179ed6d0052b206d002036 2024-11-24T09:43:32,958 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/test2727/4ed028ae71179ed6d0052b206d002036/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-24T09:43:32,959 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4ed028ae71179ed6d0052b206d002036; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63626027, jitterRate=-0.05189831554889679}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:32,960 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4ed028ae71179ed6d0052b206d002036: Writing region info on filesystem at 1732441412700Initializing all the Stores at 1732441412701 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441412701Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441412701Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441412701Obtaining lock to block concurrent updates at 1732441412811 (+110 ms)Preparing flush snapshotting stores in 4ed028ae71179ed6d0052b206d002036 at 1732441412811Finished memstore snapshotting test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732441412811Flushing stores of test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036. at 1732441412811Flushing 4ed028ae71179ed6d0052b206d002036/a: creating writer at 1732441412811Flushing 4ed028ae71179ed6d0052b206d002036/a: appending metadata at 1732441412832 (+21 ms)Flushing 4ed028ae71179ed6d0052b206d002036/a: closing flushed file at 1732441412832Flushing 4ed028ae71179ed6d0052b206d002036/b: creating writer at 1732441412847 (+15 ms)Flushing 4ed028ae71179ed6d0052b206d002036/b: appending metadata at 1732441412867 (+20 ms)Flushing 4ed028ae71179ed6d0052b206d002036/b: closing flushed file at 1732441412867Flushing 4ed028ae71179ed6d0052b206d002036/c: creating writer at 1732441412882 (+15 ms)Flushing 4ed028ae71179ed6d0052b206d002036/c: appending metadata at 1732441412909 (+27 ms)Flushing 4ed028ae71179ed6d0052b206d002036/c: closing flushed file at 1732441412909Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5949ee07: reopening flushed file at 1732441412924 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44d0d99b: reopening flushed file at 1732441412933 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38c8af3: reopening flushed file at 1732441412942 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 4ed028ae71179ed6d0052b206d002036 in 140ms, sequenceid=6000, compaction requested=false; wal=null at 1732441412951 (+9 ms)Cleaning up temporary data from old regions at 1732441412954 (+3 ms)Region opened successfully at 1732441412960 (+6 ms) 2024-11-24T09:43:32,961 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-24T09:43:32,962 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4ed028ae71179ed6d0052b206d002036, disabling compactions & flushes 2024-11-24T09:43:32,962 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036. 2024-11-24T09:43:32,962 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036. 2024-11-24T09:43:32,962 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036. after waiting 0 ms 2024-11-24T09:43:32,962 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036. 2024-11-24T09:43:32,963 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732441411933.4ed028ae71179ed6d0052b206d002036. 2024-11-24T09:43:32,963 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4ed028ae71179ed6d0052b206d002036: Waiting for close lock at 1732441412962Disabling compacts and flushes for region at 1732441412962Disabling writes for close at 1732441412962Writing region close event to WAL at 1732441412963 (+1 ms)Closed at 1732441412963 2024-11-24T09:43:32,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741870_1047 (size=95) 2024-11-24T09:43:32,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741870_1047 (size=95) 2024-11-24T09:43:32,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741870_1047 (size=95) 2024-11-24T09:43:32,969 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:32,969 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732441412674) 2024-11-24T09:43:32,980 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=398 (was 396) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:40540 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:45558 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:44910 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=7791 (was 7954) 2024-11-24T09:43:32,991 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=398, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=7789 2024-11-24T09:43:33,006 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:33,013 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:33,014 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732441413014 2024-11-24T09:43:33,022 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006/wal.1732441413014 2024-11-24T09:43:33,024 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591)] 2024-11-24T09:43:33,025 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 902cb8f39378451737705df7be2bd663, NAME => 'testSequentialEditLogSeqNum,,1732441413007.902cb8f39378451737705df7be2bd663.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:33,026 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732441413007.902cb8f39378451737705df7be2bd663.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:33,026 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 902cb8f39378451737705df7be2bd663 2024-11-24T09:43:33,026 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 902cb8f39378451737705df7be2bd663 2024-11-24T09:43:33,027 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663 doesn't exist for region: 902cb8f39378451737705df7be2bd663 on table testSequentialEditLogSeqNum 2024-11-24T09:43:33,027 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 902cb8f39378451737705df7be2bd663 on table testSequentialEditLogSeqNum 2024-11-24T09:43:33,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741875_1052 (size=62) 2024-11-24T09:43:33,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741875_1052 (size=62) 2024-11-24T09:43:33,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741875_1052 (size=62) 2024-11-24T09:43:33,042 INFO [StoreOpener-902cb8f39378451737705df7be2bd663-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 902cb8f39378451737705df7be2bd663 2024-11-24T09:43:33,043 INFO [StoreOpener-902cb8f39378451737705df7be2bd663-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 902cb8f39378451737705df7be2bd663 columnFamilyName a 2024-11-24T09:43:33,043 DEBUG [StoreOpener-902cb8f39378451737705df7be2bd663-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:33,044 INFO [StoreOpener-902cb8f39378451737705df7be2bd663-1 {}] regionserver.HStore(327): Store=902cb8f39378451737705df7be2bd663/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:33,044 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 902cb8f39378451737705df7be2bd663 2024-11-24T09:43:33,044 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663 2024-11-24T09:43:33,045 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663 2024-11-24T09:43:33,045 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 902cb8f39378451737705df7be2bd663 2024-11-24T09:43:33,045 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 902cb8f39378451737705df7be2bd663 2024-11-24T09:43:33,047 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 902cb8f39378451737705df7be2bd663 2024-11-24T09:43:33,049 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:33,049 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 902cb8f39378451737705df7be2bd663; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60000416, jitterRate=-0.10592412948608398}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:43:33,050 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 902cb8f39378451737705df7be2bd663: Writing region info on filesystem at 1732441413026Initializing all the Stores at 1732441413041 (+15 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441413041Cleaning up temporary data from old regions at 1732441413045 (+4 ms)Region opened successfully at 1732441413050 (+5 ms) 2024-11-24T09:43:33,062 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 902cb8f39378451737705df7be2bd663 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-24T09:43:33,083 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/.tmp/a/890d8eb951104456b7c38adf7248a133 is 81, key is testSequentialEditLogSeqNum/a:x0/1732441413050/Put/seqid=0 2024-11-24T09:43:33,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741876_1053 (size=5833) 2024-11-24T09:43:33,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741876_1053 (size=5833) 2024-11-24T09:43:33,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741876_1053 (size=5833) 2024-11-24T09:43:33,090 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/.tmp/a/890d8eb951104456b7c38adf7248a133 2024-11-24T09:43:33,097 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/.tmp/a/890d8eb951104456b7c38adf7248a133 as hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/a/890d8eb951104456b7c38adf7248a133 2024-11-24T09:43:33,105 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/a/890d8eb951104456b7c38adf7248a133, entries=10, sequenceid=13, filesize=5.7 K 2024-11-24T09:43:33,106 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 902cb8f39378451737705df7be2bd663 in 44ms, sequenceid=13, compaction requested=false 2024-11-24T09:43:33,107 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 902cb8f39378451737705df7be2bd663: 2024-11-24T09:43:33,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T09:43:33,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T09:43:33,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T09:43:33,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T09:43:33,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T09:43:33,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741874_1051 (size=1844) 2024-11-24T09:43:33,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741874_1051 (size=1844) 2024-11-24T09:43:33,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741874_1051 (size=1844) 2024-11-24T09:43:33,139 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006/wal.1732441413014, size=1.8 K (1844bytes) 2024-11-24T09:43:33,139 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006/wal.1732441413014 2024-11-24T09:43:33,140 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006/wal.1732441413014 after 1ms 2024-11-24T09:43:33,143 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006/wal.1732441413014: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:33,143 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006/wal.1732441413014 took 4ms 2024-11-24T09:43:33,147 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006/wal.1732441413014 so closing down 2024-11-24T09:43:33,147 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:33,148 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732441413014.temp 2024-11-24T09:43:33,150 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/recovered.edits/0000000000000000003-wal.1732441413014.temp 2024-11-24T09:43:33,150 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:33,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741877_1054 (size=1477) 2024-11-24T09:43:33,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741877_1054 (size=1477) 2024-11-24T09:43:33,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741877_1054 (size=1477) 2024-11-24T09:43:33,158 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/recovered.edits/0000000000000000003-wal.1732441413014.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T09:43:33,160 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/recovered.edits/0000000000000000003-wal.1732441413014.temp to hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/recovered.edits/0000000000000000020 2024-11-24T09:43:33,160 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:36549/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006/wal.1732441413014, size=1.8 K, length=1844, corrupted=false, cancelled=false 2024-11-24T09:43:33,160 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006/wal.1732441413014, journal: Splitting hdfs://localhost:36549/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006/wal.1732441413014, size=1.8 K (1844bytes) at 1732441413139Finishing writing output for hdfs://localhost:36549/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006/wal.1732441413014 so closing down at 1732441413147 (+8 ms)Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/recovered.edits/0000000000000000003-wal.1732441413014.temp at 1732441413150 (+3 ms)3 split writer threads finished at 1732441413150Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/recovered.edits/0000000000000000003-wal.1732441413014.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732441413158 (+8 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/recovered.edits/0000000000000000003-wal.1732441413014.temp to hdfs://localhost:36549/hbase/data/default/testSequentialEditLogSeqNum/902cb8f39378451737705df7be2bd663/recovered.edits/0000000000000000020 at 1732441413160 (+2 ms)Processed 17 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:36549/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441413006/wal.1732441413014, size=1.8 K, length=1844, corrupted=false, cancelled=false at 1732441413160 2024-11-24T09:43:33,172 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=403 (was 398) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:45558 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:44910 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=308 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=7771 (was 7789) 2024-11-24T09:43:33,182 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=403, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=7770 2024-11-24T09:43:33,199 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:33,201 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:33,231 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:33,235 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73553837, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/hregion-73553837, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:33,251 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73553837/hregion-73553837.1732441413236, exclude list is [], retry=0 2024-11-24T09:43:33,254 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:33,254 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:33,255 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:33,256 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73553837/hregion-73553837.1732441413236 2024-11-24T09:43:33,257 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591)] 2024-11-24T09:43:33,257 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a30c49babffcb128a6c0d0c562b609c1, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36549/hbase 2024-11-24T09:43:33,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741879_1056 (size=70) 2024-11-24T09:43:33,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741879_1056 (size=70) 2024-11-24T09:43:33,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741879_1056 (size=70) 2024-11-24T09:43:33,267 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:33,268 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,270 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a30c49babffcb128a6c0d0c562b609c1 columnFamilyName a 2024-11-24T09:43:33,270 DEBUG [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:33,270 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(327): Store=a30c49babffcb128a6c0d0c562b609c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:33,271 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,272 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a30c49babffcb128a6c0d0c562b609c1 columnFamilyName b 2024-11-24T09:43:33,272 DEBUG [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:33,273 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(327): Store=a30c49babffcb128a6c0d0c562b609c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:33,273 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,275 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a30c49babffcb128a6c0d0c562b609c1 columnFamilyName c 2024-11-24T09:43:33,275 DEBUG [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:33,275 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(327): Store=a30c49babffcb128a6c0d0c562b609c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:33,275 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,276 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,277 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,278 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,278 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,279 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:33,280 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,283 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:33,283 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a30c49babffcb128a6c0d0c562b609c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66516299, jitterRate=-0.008829906582832336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:33,285 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a30c49babffcb128a6c0d0c562b609c1: Writing region info on filesystem at 1732441413267Initializing all the Stores at 1732441413268 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441413268Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441413268Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441413268Cleaning up temporary data from old regions at 1732441413278 (+10 ms)Region opened successfully at 1732441413284 (+6 ms) 2024-11-24T09:43:33,285 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a30c49babffcb128a6c0d0c562b609c1, disabling compactions & flushes 2024-11-24T09:43:33,285 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1. 2024-11-24T09:43:33,285 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1. 2024-11-24T09:43:33,285 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1. after waiting 0 ms 2024-11-24T09:43:33,285 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1. 2024-11-24T09:43:33,285 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1. 2024-11-24T09:43:33,286 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a30c49babffcb128a6c0d0c562b609c1: Waiting for close lock at 1732441413285Disabling compacts and flushes for region at 1732441413285Disabling writes for close at 1732441413285Writing region close event to WAL at 1732441413285Closed at 1732441413285 2024-11-24T09:43:33,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741878_1055 (size=95) 2024-11-24T09:43:33,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741878_1055 (size=95) 2024-11-24T09:43:33,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741878_1055 (size=95) 2024-11-24T09:43:33,291 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:33,291 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73553837:(num 1732441413236) 2024-11-24T09:43:33,291 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:33,293 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:33,308 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294, exclude list is [], retry=0 2024-11-24T09:43:33,311 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:33,312 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:33,312 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:33,314 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294 2024-11-24T09:43:33,314 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:35185:35185)] 2024-11-24T09:43:33,315 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a30c49babffcb128a6c0d0c562b609c1, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:33,315 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:33,315 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,315 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,318 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,320 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a30c49babffcb128a6c0d0c562b609c1 columnFamilyName a 2024-11-24T09:43:33,320 DEBUG [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:33,321 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(327): Store=a30c49babffcb128a6c0d0c562b609c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:33,321 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,323 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a30c49babffcb128a6c0d0c562b609c1 columnFamilyName b 2024-11-24T09:43:33,323 DEBUG [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:33,323 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(327): Store=a30c49babffcb128a6c0d0c562b609c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:33,324 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,325 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a30c49babffcb128a6c0d0c562b609c1 columnFamilyName c 2024-11-24T09:43:33,325 DEBUG [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:33,326 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(327): Store=a30c49babffcb128a6c0d0c562b609c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:33,326 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,327 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,328 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,330 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,330 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,330 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:33,332 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:33,332 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a30c49babffcb128a6c0d0c562b609c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61472348, jitterRate=-0.08399063348770142}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:33,334 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a30c49babffcb128a6c0d0c562b609c1: Writing region info on filesystem at 1732441413315Initializing all the Stores at 1732441413316 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441413316Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441413318 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441413318Cleaning up temporary data from old regions at 1732441413330 (+12 ms)Region opened successfully at 1732441413334 (+4 ms) 2024-11-24T09:43:33,338 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732441413337/Put/seqid=0 2024-11-24T09:43:33,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741881_1058 (size=4826) 2024-11-24T09:43:33,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741881_1058 (size=4826) 2024-11-24T09:43:33,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741881_1058 (size=4826) 2024-11-24T09:43:33,350 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36549/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in a30c49babffcb128a6c0d0c562b609c1/a 2024-11-24T09:43:33,358 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-24T09:43:33,359 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T09:43:33,359 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a30c49babffcb128a6c0d0c562b609c1: 2024-11-24T09:43:33,361 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/a/a0f5d9c6f31b4fedb2aeb7b0becacb91_SeqId_3_ 2024-11-24T09:43:33,362 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36549/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into a30c49babffcb128a6c0d0c562b609c1/a as hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/a/a0f5d9c6f31b4fedb2aeb7b0becacb91_SeqId_3_ - updating store file list. 2024-11-24T09:43:33,369 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for a0f5d9c6f31b4fedb2aeb7b0becacb91_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T09:43:33,370 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/a/a0f5d9c6f31b4fedb2aeb7b0becacb91_SeqId_3_ into a30c49babffcb128a6c0d0c562b609c1/a 2024-11-24T09:43:33,370 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36549/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into a30c49babffcb128a6c0d0c562b609c1/a (new location: hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/a/a0f5d9c6f31b4fedb2aeb7b0becacb91_SeqId_3_) 2024-11-24T09:43:33,420 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294, size=0 (0bytes) 2024-11-24T09:43:33,420 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294 might be still open, length is 0 2024-11-24T09:43:33,420 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294 2024-11-24T09:43:33,421 WARN [IPC Server handler 0 on default port 36549 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-24T09:43:33,421 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294 after 0ms 2024-11-24T09:43:33,524 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:40638 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40638 dst: /127.0.0.1:33653 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33653 remote=/127.0.0.1:40638]. Total timeout mills is 60000, 59852 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:33,525 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:37292 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37953:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37292 dst: /127.0.0.1:37953 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:33,525 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:45672 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37499:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45672 dst: /127.0.0.1:37499 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:33,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741880_1059 (size=471) 2024-11-24T09:43:33,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741880_1059 (size=471) 2024-11-24T09:43:37,423 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294 after 4002ms 2024-11-24T09:43:37,431 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:37,432 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294 took 4012ms 2024-11-24T09:43:37,436 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294; continuing. 2024-11-24T09:43:37,436 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294 so closing down 2024-11-24T09:43:37,436 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:37,438 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732441413294.temp 2024-11-24T09:43:37,440 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005-wal.1732441413294.temp 2024-11-24T09:43:37,440 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:37,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741882_1060 (size=259) 2024-11-24T09:43:37,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741882_1060 (size=259) 2024-11-24T09:43:37,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741882_1060 (size=259) 2024-11-24T09:43:37,448 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005-wal.1732441413294.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T09:43:37,449 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005-wal.1732441413294.temp to hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005 2024-11-24T09:43:37,449 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T09:43:37,449 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294, journal: Splitting hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294, size=0 (0bytes) at 1732441413420Finishing writing output for hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294 so closing down at 1732441417436 (+4016 ms)Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005-wal.1732441413294.temp at 1732441417440 (+4 ms)3 split writer threads finished at 1732441417440Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005-wal.1732441413294.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732441417448 (+8 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005-wal.1732441413294.temp to hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005 at 1732441417449 (+1 ms)Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294, size=0, length=0, corrupted=false, cancelled=false at 1732441417449 2024-11-24T09:43:37,450 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294 to hdfs://localhost:36549/hbase/oldWALs/wal.1732441413294 2024-11-24T09:43:37,451 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005 2024-11-24T09:43:37,451 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:37,453 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:37,488 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441417453, exclude list is [], retry=0 2024-11-24T09:43:37,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:37,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:37,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:37,493 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441417453 2024-11-24T09:43:37,493 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:37,494 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a30c49babffcb128a6c0d0c562b609c1, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:37,494 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:37,494 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:37,494 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:37,496 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:37,496 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a30c49babffcb128a6c0d0c562b609c1 columnFamilyName a 2024-11-24T09:43:37,496 DEBUG [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:37,503 DEBUG [StoreFileOpener-a30c49babffcb128a6c0d0c562b609c1-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for a0f5d9c6f31b4fedb2aeb7b0becacb91_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T09:43:37,503 DEBUG [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/a/a0f5d9c6f31b4fedb2aeb7b0becacb91_SeqId_3_ 2024-11-24T09:43:37,503 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(327): Store=a30c49babffcb128a6c0d0c562b609c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:37,503 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:37,504 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a30c49babffcb128a6c0d0c562b609c1 columnFamilyName b 2024-11-24T09:43:37,504 DEBUG [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:37,505 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(327): Store=a30c49babffcb128a6c0d0c562b609c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:37,505 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:37,506 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a30c49babffcb128a6c0d0c562b609c1 columnFamilyName c 2024-11-24T09:43:37,506 DEBUG [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:37,506 INFO [StoreOpener-a30c49babffcb128a6c0d0c562b609c1-1 {}] regionserver.HStore(327): Store=a30c49babffcb128a6c0d0c562b609c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:37,506 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:37,507 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:37,509 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:37,510 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005 2024-11-24T09:43:37,513 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:37,514 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005 2024-11-24T09:43:37,514 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a30c49babffcb128a6c0d0c562b609c1 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-24T09:43:37,529 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/.tmp/a/bfdf592beca049deb9622f14b3ce1a35 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732441413375/Put/seqid=0 2024-11-24T09:43:37,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741884_1062 (size=5149) 2024-11-24T09:43:37,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741884_1062 (size=5149) 2024-11-24T09:43:37,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741884_1062 (size=5149) 2024-11-24T09:43:37,537 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/.tmp/a/bfdf592beca049deb9622f14b3ce1a35 2024-11-24T09:43:37,544 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/.tmp/a/bfdf592beca049deb9622f14b3ce1a35 as hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/a/bfdf592beca049deb9622f14b3ce1a35 2024-11-24T09:43:37,551 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/a/bfdf592beca049deb9622f14b3ce1a35, entries=1, sequenceid=5, filesize=5.0 K 2024-11-24T09:43:37,551 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for a30c49babffcb128a6c0d0c562b609c1 in 37ms, sequenceid=5, compaction requested=false; wal=null 2024-11-24T09:43:37,552 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/0000000000000000005 2024-11-24T09:43:37,553 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:37,553 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:37,554 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:37,556 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a30c49babffcb128a6c0d0c562b609c1 2024-11-24T09:43:37,558 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a30c49babffcb128a6c0d0c562b609c1/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T09:43:37,559 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a30c49babffcb128a6c0d0c562b609c1; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62917850, jitterRate=-0.06245097517967224}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:37,559 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a30c49babffcb128a6c0d0c562b609c1: Writing region info on filesystem at 1732441417494Initializing all the Stores at 1732441417495 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441417495Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441417495Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441417495Obtaining lock to block concurrent updates at 1732441417514 (+19 ms)Preparing flush snapshotting stores in a30c49babffcb128a6c0d0c562b609c1 at 1732441417514Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732441417514Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1. at 1732441417514Flushing a30c49babffcb128a6c0d0c562b609c1/a: creating writer at 1732441417514Flushing a30c49babffcb128a6c0d0c562b609c1/a: appending metadata at 1732441417529 (+15 ms)Flushing a30c49babffcb128a6c0d0c562b609c1/a: closing flushed file at 1732441417529Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d95a34a: reopening flushed file at 1732441417543 (+14 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for a30c49babffcb128a6c0d0c562b609c1 in 37ms, sequenceid=5, compaction requested=false; wal=null at 1732441417551 (+8 ms)Cleaning up temporary data from old regions at 1732441417553 (+2 ms)Region opened successfully at 1732441417559 (+6 ms) 2024-11-24T09:43:37,563 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a30c49babffcb128a6c0d0c562b609c1, disabling compactions & flushes 2024-11-24T09:43:37,563 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1. 2024-11-24T09:43:37,563 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1. 2024-11-24T09:43:37,563 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1. after waiting 0 ms 2024-11-24T09:43:37,563 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1. 2024-11-24T09:43:37,564 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732441413200.a30c49babffcb128a6c0d0c562b609c1. 2024-11-24T09:43:37,564 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a30c49babffcb128a6c0d0c562b609c1: Waiting for close lock at 1732441417563Disabling compacts and flushes for region at 1732441417563Disabling writes for close at 1732441417563Writing region close event to WAL at 1732441417564 (+1 ms)Closed at 1732441417564 2024-11-24T09:43:37,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741883_1061 (size=95) 2024-11-24T09:43:37,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741883_1061 (size=95) 2024-11-24T09:43:37,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741883_1061 (size=95) 2024-11-24T09:43:37,570 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:37,570 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732441417453) 2024-11-24T09:43:37,582 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=406 (was 403) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:36549 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_832288947_22 at /127.0.0.1:37328 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_832288947_22 at /127.0.0.1:40646 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_832288947_22 at /127.0.0.1:45702 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:36549 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=308 (was 308), ProcessCount=11 (was 11), AvailableMemoryMB=7671 (was 7770) 2024-11-24T09:43:37,592 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=406, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=7670 2024-11-24T09:43:37,606 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:37,610 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T09:43:37,613 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ba35ada4783f,39325,1732441402377 2024-11-24T09:43:37,615 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ed9da91 2024-11-24T09:43:37,616 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T09:43:37,618 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T09:43:37,622 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T09:43:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T09:43:37,632 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T09:43:37,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-24T09:43:37,634 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:37,637 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T09:43:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T09:43:37,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741885_1063 (size=694) 2024-11-24T09:43:37,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741885_1063 (size=694) 2024-11-24T09:43:37,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741885_1063 (size=694) 2024-11-24T09:43:37,655 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f1ee6750d931ca2321055907c01bc718, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88 2024-11-24T09:43:37,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741886_1064 (size=77) 2024-11-24T09:43:37,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741886_1064 (size=77) 2024-11-24T09:43:37,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741886_1064 (size=77) 2024-11-24T09:43:37,666 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:37,666 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing f1ee6750d931ca2321055907c01bc718, disabling compactions & flushes 2024-11-24T09:43:37,666 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:37,666 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:37,666 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. after waiting 0 ms 2024-11-24T09:43:37,666 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:37,666 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:37,666 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for f1ee6750d931ca2321055907c01bc718: Waiting for close lock at 1732441417666Disabling compacts and flushes for region at 1732441417666Disabling writes for close at 1732441417666Writing region close event to WAL at 1732441417666Closed at 1732441417666 2024-11-24T09:43:37,668 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T09:43:37,672 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732441417668"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732441417668"}]},"ts":"1732441417668"} 2024-11-24T09:43:37,675 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T09:43:37,677 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T09:43:37,679 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732441417677"}]},"ts":"1732441417677"} 2024-11-24T09:43:37,683 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-24T09:43:37,683 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {ba35ada4783f=0} racks are {/default-rack=0} 2024-11-24T09:43:37,685 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T09:43:37,685 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T09:43:37,685 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T09:43:37,685 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T09:43:37,685 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T09:43:37,685 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T09:43:37,685 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T09:43:37,685 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T09:43:37,685 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T09:43:37,685 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T09:43:37,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, ASSIGN}] 2024-11-24T09:43:37,688 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, ASSIGN 2024-11-24T09:43:37,690 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, ASSIGN; state=OFFLINE, location=ba35ada4783f,46517,1732441403295; forceNewPlan=false, retain=false 2024-11-24T09:43:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T09:43:37,843 INFO [ba35ada4783f:39325 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T09:43:37,844 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f1ee6750d931ca2321055907c01bc718, regionState=OPENING, regionLocation=ba35ada4783f,46517,1732441403295 2024-11-24T09:43:37,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, ASSIGN because future has completed 2024-11-24T09:43:37,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,46517,1732441403295}] 2024-11-24T09:43:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T09:43:38,014 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,015 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f1ee6750d931ca2321055907c01bc718, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:38,016 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,017 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:38,017 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,018 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,020 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,022 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1ee6750d931ca2321055907c01bc718 columnFamilyName cf1 2024-11-24T09:43:38,022 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:38,023 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(327): Store=f1ee6750d931ca2321055907c01bc718/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:38,023 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,024 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1ee6750d931ca2321055907c01bc718 columnFamilyName cf2 2024-11-24T09:43:38,024 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:38,025 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(327): Store=f1ee6750d931ca2321055907c01bc718/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:38,025 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,026 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,026 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,027 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,027 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,028 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T09:43:38,029 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,032 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:38,032 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f1ee6750d931ca2321055907c01bc718; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63047648, jitterRate=-0.0605168342590332}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T09:43:38,032 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,033 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f1ee6750d931ca2321055907c01bc718: Running coprocessor pre-open hook at 1732441418018Writing region info on filesystem at 1732441418018Initializing all the Stores at 1732441418019 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441418020 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441418020Cleaning up temporary data from old regions at 1732441418027 (+7 ms)Running coprocessor post-open hooks at 1732441418032 (+5 ms)Region opened successfully at 1732441418033 (+1 ms) 2024-11-24T09:43:38,034 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., pid=6, masterSystemTime=1732441418005 2024-11-24T09:43:38,038 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,038 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,039 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f1ee6750d931ca2321055907c01bc718, regionState=OPEN, openSeqNum=2, regionLocation=ba35ada4783f,46517,1732441403295 2024-11-24T09:43:38,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,46517,1732441403295 because future has completed 2024-11-24T09:43:38,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T09:43:38,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,46517,1732441403295 in 193 msec 2024-11-24T09:43:38,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T09:43:38,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, ASSIGN in 361 msec 2024-11-24T09:43:38,052 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T09:43:38,052 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732441418052"}]},"ts":"1732441418052"} 2024-11-24T09:43:38,056 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-24T09:43:38,058 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T09:43:38,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 434 msec 2024-11-24T09:43:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T09:43:38,270 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-24T09:43:38,270 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-24T09:43:38,271 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T09:43:38,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-24T09:43:38,277 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T09:43:38,277 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-24T09:43:38,288 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=2] 2024-11-24T09:43:38,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=f1ee6750d931ca2321055907c01bc718, source=ba35ada4783f,46517,1732441403295, destination=ba35ada4783f,42075,1732441403106, warming up region on ba35ada4783f,42075,1732441403106 2024-11-24T09:43:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T09:43:38,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=f1ee6750d931ca2321055907c01bc718, source=ba35ada4783f,46517,1732441403295, destination=ba35ada4783f,42075,1732441403106, running balancer 2024-11-24T09:43:38,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54481, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T09:43:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, REOPEN/MOVE 2024-11-24T09:43:38,307 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, REOPEN/MOVE 2024-11-24T09:43:38,311 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f1ee6750d931ca2321055907c01bc718, regionState=CLOSING, regionLocation=ba35ada4783f,46517,1732441403295 2024-11-24T09:43:38,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(7855): Warmup {ENCODED => f1ee6750d931ca2321055907c01bc718, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:38,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, REOPEN/MOVE because future has completed 2024-11-24T09:43:38,319 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,320 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1ee6750d931ca2321055907c01bc718 columnFamilyName cf1 2024-11-24T09:43:38,320 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:38,321 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(327): Store=f1ee6750d931ca2321055907c01bc718/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:38,321 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,323 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1ee6750d931ca2321055907c01bc718 columnFamilyName cf2 2024-11-24T09:43:38,323 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:38,323 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T09:43:38,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,46517,1732441403295}] 2024-11-24T09:43:38,323 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(327): Store=f1ee6750d931ca2321055907c01bc718/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1722): Closing f1ee6750d931ca2321055907c01bc718, disabling compactions & flushes 2024-11-24T09:43:38,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. after waiting 0 ms 2024-11-24T09:43:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1676): Region close journal for f1ee6750d931ca2321055907c01bc718: Waiting for close lock at 1732441418324Disabling compacts and flushes for region at 1732441418324Disabling writes for close at 1732441418324Writing region close event to WAL at 1732441418327 (+3 ms)Closed at 1732441418327 2024-11-24T09:43:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T09:43:38,485 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,485 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T09:43:38,486 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing f1ee6750d931ca2321055907c01bc718, disabling compactions & flushes 2024-11-24T09:43:38,486 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,487 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,487 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. after waiting 0 ms 2024-11-24T09:43:38,487 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,487 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing f1ee6750d931ca2321055907c01bc718 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-24T09:43:38,510 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp/cf1/bd0ba39307e944b2b2d7020b6b99f877 is 35, key is r1/cf1:q/1732441418290/Put/seqid=0 2024-11-24T09:43:38,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741887_1065 (size=4783) 2024-11-24T09:43:38,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741887_1065 (size=4783) 2024-11-24T09:43:38,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741887_1065 (size=4783) 2024-11-24T09:43:38,528 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp/cf1/bd0ba39307e944b2b2d7020b6b99f877 2024-11-24T09:43:38,535 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp/cf1/bd0ba39307e944b2b2d7020b6b99f877 as hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/bd0ba39307e944b2b2d7020b6b99f877 2024-11-24T09:43:38,542 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/bd0ba39307e944b2b2d7020b6b99f877, entries=1, sequenceid=5, filesize=4.7 K 2024-11-24T09:43:38,544 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for f1ee6750d931ca2321055907c01bc718 in 57ms, sequenceid=5, compaction requested=false 2024-11-24T09:43:38,544 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-24T09:43:38,549 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T09:43:38,551 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,551 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for f1ee6750d931ca2321055907c01bc718: Waiting for close lock at 1732441418486Running coprocessor pre-close hooks at 1732441418486Disabling compacts and flushes for region at 1732441418486Disabling writes for close at 1732441418487 (+1 ms)Obtaining lock to block concurrent updates at 1732441418488 (+1 ms)Preparing flush snapshotting stores in f1ee6750d931ca2321055907c01bc718 at 1732441418488Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732441418488Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. at 1732441418490 (+2 ms)Flushing f1ee6750d931ca2321055907c01bc718/cf1: creating writer at 1732441418490Flushing f1ee6750d931ca2321055907c01bc718/cf1: appending metadata at 1732441418509 (+19 ms)Flushing f1ee6750d931ca2321055907c01bc718/cf1: closing flushed file at 1732441418509Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@120b227b: reopening flushed file at 1732441418534 (+25 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for f1ee6750d931ca2321055907c01bc718 in 57ms, sequenceid=5, compaction requested=false at 1732441418544 (+10 ms)Writing region close event to WAL at 1732441418545 (+1 ms)Running coprocessor post-close hooks at 1732441418550 (+5 ms)Closed at 1732441418551 (+1 ms) 2024-11-24T09:43:38,552 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding f1ee6750d931ca2321055907c01bc718 move to ba35ada4783f,42075,1732441403106 record at close sequenceid=5 2024-11-24T09:43:38,556 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f1ee6750d931ca2321055907c01bc718, regionState=CLOSED 2024-11-24T09:43:38,556 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,46517,1732441403295 because future has completed 2024-11-24T09:43:38,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T09:43:38,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,46517,1732441403295 in 237 msec 2024-11-24T09:43:38,565 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, REOPEN/MOVE; state=CLOSED, location=ba35ada4783f,42075,1732441403106; forceNewPlan=false, retain=false 2024-11-24T09:43:38,715 INFO [ba35ada4783f:39325 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T09:43:38,716 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f1ee6750d931ca2321055907c01bc718, regionState=OPENING, regionLocation=ba35ada4783f,42075,1732441403106 2024-11-24T09:43:38,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, REOPEN/MOVE because future has completed 2024-11-24T09:43:38,719 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,42075,1732441403106}] 2024-11-24T09:43:38,880 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,880 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => f1ee6750d931ca2321055907c01bc718, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:38,881 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,881 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:38,881 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,881 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,884 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,886 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1ee6750d931ca2321055907c01bc718 columnFamilyName cf1 2024-11-24T09:43:38,886 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:38,895 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/bd0ba39307e944b2b2d7020b6b99f877 2024-11-24T09:43:38,896 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(327): Store=f1ee6750d931ca2321055907c01bc718/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:38,896 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,897 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1ee6750d931ca2321055907c01bc718 columnFamilyName cf2 2024-11-24T09:43:38,897 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:38,897 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(327): Store=f1ee6750d931ca2321055907c01bc718/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:38,897 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,898 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,900 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,900 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,901 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,901 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T09:43:38,903 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,904 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened f1ee6750d931ca2321055907c01bc718; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67234810, jitterRate=0.001876741647720337}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T09:43:38,904 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:38,905 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for f1ee6750d931ca2321055907c01bc718: Running coprocessor pre-open hook at 1732441418882Writing region info on filesystem at 1732441418882Initializing all the Stores at 1732441418884 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441418884Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441418884Cleaning up temporary data from old regions at 1732441418901 (+17 ms)Running coprocessor post-open hooks at 1732441418904 (+3 ms)Region opened successfully at 1732441418905 (+1 ms) 2024-11-24T09:43:38,906 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., pid=9, masterSystemTime=1732441418872 2024-11-24T09:43:38,908 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,908 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:38,909 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f1ee6750d931ca2321055907c01bc718, regionState=OPEN, openSeqNum=9, regionLocation=ba35ada4783f,42075,1732441403106 2024-11-24T09:43:38,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,42075,1732441403106 because future has completed 2024-11-24T09:43:38,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T09:43:38,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,42075,1732441403106 in 194 msec 2024-11-24T09:43:38,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, REOPEN/MOVE in 609 msec 2024-11-24T09:43:38,967 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T09:43:38,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T09:43:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46517 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:44968 deadline: 1732441478974, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=42075 startCode=1732441403106. As of locationSeqNum=5. 2024-11-24T09:43:39,002 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=42075 startCode=1732441403106. As of locationSeqNum=5. 2024-11-24T09:43:39,002 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=42075 startCode=1732441403106. As of locationSeqNum=5. 2024-11-24T09:43:39,002 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,42075,1732441403106, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=42075 startCode=1732441403106. As of locationSeqNum=5. 2024-11-24T09:43:39,108 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T09:43:39,110 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T09:43:39,121 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f1ee6750d931ca2321055907c01bc718 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-24T09:43:39,139 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp/cf1/f20bdb7f81b349048f40ac4f267d6c7f is 29, key is r1/cf1:/1732441419111/DeleteFamily/seqid=0 2024-11-24T09:43:39,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741888_1066 (size=4906) 2024-11-24T09:43:39,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741888_1066 (size=4906) 2024-11-24T09:43:39,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741888_1066 (size=4906) 2024-11-24T09:43:39,149 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp/cf1/f20bdb7f81b349048f40ac4f267d6c7f 2024-11-24T09:43:39,158 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f20bdb7f81b349048f40ac4f267d6c7f 2024-11-24T09:43:39,179 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp/cf2/aa659dedc89843c999d832cfbea4c1af is 29, key is r1/cf2:/1732441419111/DeleteFamily/seqid=0 2024-11-24T09:43:39,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741889_1067 (size=4906) 2024-11-24T09:43:39,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741889_1067 (size=4906) 2024-11-24T09:43:39,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741889_1067 (size=4906) 2024-11-24T09:43:39,187 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp/cf2/aa659dedc89843c999d832cfbea4c1af 2024-11-24T09:43:39,194 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aa659dedc89843c999d832cfbea4c1af 2024-11-24T09:43:39,195 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp/cf1/f20bdb7f81b349048f40ac4f267d6c7f as hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/f20bdb7f81b349048f40ac4f267d6c7f 2024-11-24T09:43:39,204 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f20bdb7f81b349048f40ac4f267d6c7f 2024-11-24T09:43:39,205 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/f20bdb7f81b349048f40ac4f267d6c7f, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T09:43:39,206 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp/cf2/aa659dedc89843c999d832cfbea4c1af as hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf2/aa659dedc89843c999d832cfbea4c1af 2024-11-24T09:43:39,213 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aa659dedc89843c999d832cfbea4c1af 2024-11-24T09:43:39,214 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf2/aa659dedc89843c999d832cfbea4c1af, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T09:43:39,216 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for f1ee6750d931ca2321055907c01bc718 in 95ms, sequenceid=12, compaction requested=false 2024-11-24T09:43:39,216 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f1ee6750d931ca2321055907c01bc718: 2024-11-24T09:43:39,219 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T09:43:39,220 DEBUG [Time-limited test {}] regionserver.HStore(1541): f1ee6750d931ca2321055907c01bc718/cf1 is initiating major compaction (all files) 2024-11-24T09:43:39,220 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T09:43:39,220 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:39,221 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of f1ee6750d931ca2321055907c01bc718/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,221 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/bd0ba39307e944b2b2d7020b6b99f877, hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/f20bdb7f81b349048f40ac4f267d6c7f] into tmpdir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp, totalSize=9.5 K 2024-11-24T09:43:39,223 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bd0ba39307e944b2b2d7020b6b99f877, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732441418290 2024-11-24T09:43:39,223 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f20bdb7f81b349048f40ac4f267d6c7f, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T09:43:39,237 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): f1ee6750d931ca2321055907c01bc718#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T09:43:39,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741890_1068 (size=4626) 2024-11-24T09:43:39,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741890_1068 (size=4626) 2024-11-24T09:43:39,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741890_1068 (size=4626) 2024-11-24T09:43:39,254 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp/cf1/b84e24fb63bf40a0bcd4d34c2b222161 as hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/b84e24fb63bf40a0bcd4d34c2b222161 2024-11-24T09:43:39,271 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in f1ee6750d931ca2321055907c01bc718/cf1 of f1ee6750d931ca2321055907c01bc718 into b84e24fb63bf40a0bcd4d34c2b222161(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T09:43:39,271 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for f1ee6750d931ca2321055907c01bc718: 2024-11-24T09:43:39,272 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-24T09:43:39,272 DEBUG [Time-limited test {}] regionserver.HStore(1541): f1ee6750d931ca2321055907c01bc718/cf2 is initiating major compaction (all files) 2024-11-24T09:43:39,272 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T09:43:39,272 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:43:39,272 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of f1ee6750d931ca2321055907c01bc718/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,272 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf2/aa659dedc89843c999d832cfbea4c1af] into tmpdir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp, totalSize=4.8 K 2024-11-24T09:43:39,273 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting aa659dedc89843c999d832cfbea4c1af, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T09:43:39,282 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): f1ee6750d931ca2321055907c01bc718#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T09:43:39,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741891_1069 (size=4592) 2024-11-24T09:43:39,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741891_1069 (size=4592) 2024-11-24T09:43:39,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741891_1069 (size=4592) 2024-11-24T09:43:39,304 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/.tmp/cf2/8fb57b3ebfbf4b3e8da63bf469eb3c14 as hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf2/8fb57b3ebfbf4b3e8da63bf469eb3c14 2024-11-24T09:43:39,314 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in f1ee6750d931ca2321055907c01bc718/cf2 of f1ee6750d931ca2321055907c01bc718 into 8fb57b3ebfbf4b3e8da63bf469eb3c14(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T09:43:39,314 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for f1ee6750d931ca2321055907c01bc718: 2024-11-24T09:43:39,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=f1ee6750d931ca2321055907c01bc718, source=ba35ada4783f,42075,1732441403106, destination=ba35ada4783f,46517,1732441403295, warming up region on ba35ada4783f,46517,1732441403295 2024-11-24T09:43:39,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=f1ee6750d931ca2321055907c01bc718, source=ba35ada4783f,42075,1732441403106, destination=ba35ada4783f,46517,1732441403295, running balancer 2024-11-24T09:43:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, REOPEN/MOVE 2024-11-24T09:43:39,322 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, REOPEN/MOVE 2024-11-24T09:43:39,324 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f1ee6750d931ca2321055907c01bc718, regionState=CLOSING, regionLocation=ba35ada4783f,42075,1732441403106 2024-11-24T09:43:39,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46517 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46517 {}] regionserver.HRegion(7855): Warmup {ENCODED => f1ee6750d931ca2321055907c01bc718, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46517 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:39,326 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,327 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1ee6750d931ca2321055907c01bc718 columnFamilyName cf1 2024-11-24T09:43:39,327 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:39,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, REOPEN/MOVE because future has completed 2024-11-24T09:43:39,329 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T09:43:39,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,42075,1732441403106}] 2024-11-24T09:43:39,336 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/b84e24fb63bf40a0bcd4d34c2b222161 2024-11-24T09:43:39,343 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/bd0ba39307e944b2b2d7020b6b99f877 2024-11-24T09:43:39,348 INFO [StoreFileOpener-f1ee6750d931ca2321055907c01bc718-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f20bdb7f81b349048f40ac4f267d6c7f 2024-11-24T09:43:39,348 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/f20bdb7f81b349048f40ac4f267d6c7f 2024-11-24T09:43:39,348 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(327): Store=f1ee6750d931ca2321055907c01bc718/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:39,348 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,349 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1ee6750d931ca2321055907c01bc718 columnFamilyName cf2 2024-11-24T09:43:39,350 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:39,357 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf2/8fb57b3ebfbf4b3e8da63bf469eb3c14 2024-11-24T09:43:39,363 INFO [StoreFileOpener-f1ee6750d931ca2321055907c01bc718-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aa659dedc89843c999d832cfbea4c1af 2024-11-24T09:43:39,363 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf2/aa659dedc89843c999d832cfbea4c1af 2024-11-24T09:43:39,363 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(327): Store=f1ee6750d931ca2321055907c01bc718/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46517 {}] regionserver.HRegion(1722): Closing f1ee6750d931ca2321055907c01bc718, disabling compactions & flushes 2024-11-24T09:43:39,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46517 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46517 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46517 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. after waiting 0 ms 2024-11-24T09:43:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46517 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46517 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46517 {}] regionserver.HRegion(1676): Region close journal for f1ee6750d931ca2321055907c01bc718: Waiting for close lock at 1732441419363Disabling compacts and flushes for region at 1732441419363Disabling writes for close at 1732441419364 (+1 ms)Writing region close event to WAL at 1732441419365 (+1 ms)Closed at 1732441419365 2024-11-24T09:43:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-24T09:43:39,483 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,483 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T09:43:39,483 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing f1ee6750d931ca2321055907c01bc718, disabling compactions & flushes 2024-11-24T09:43:39,483 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,484 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,484 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. after waiting 0 ms 2024-11-24T09:43:39,484 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,484 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/bd0ba39307e944b2b2d7020b6b99f877, hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/f20bdb7f81b349048f40ac4f267d6c7f] to archive 2024-11-24T09:43:39,487 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T09:43:39,491 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/bd0ba39307e944b2b2d7020b6b99f877 to hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/bd0ba39307e944b2b2d7020b6b99f877 2024-11-24T09:43:39,493 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/f20bdb7f81b349048f40ac4f267d6c7f to hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/f20bdb7f81b349048f40ac4f267d6c7f 2024-11-24T09:43:39,504 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf2/aa659dedc89843c999d832cfbea4c1af] to archive 2024-11-24T09:43:39,505 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T09:43:39,507 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf2/aa659dedc89843c999d832cfbea4c1af to hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf2/aa659dedc89843c999d832cfbea4c1af 2024-11-24T09:43:39,512 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-24T09:43:39,513 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,513 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for f1ee6750d931ca2321055907c01bc718: Waiting for close lock at 1732441419483Running coprocessor pre-close hooks at 1732441419483Disabling compacts and flushes for region at 1732441419483Disabling writes for close at 1732441419484 (+1 ms)Writing region close event to WAL at 1732441419509 (+25 ms)Running coprocessor post-close hooks at 1732441419513 (+4 ms)Closed at 1732441419513 2024-11-24T09:43:39,513 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding f1ee6750d931ca2321055907c01bc718 move to ba35ada4783f,46517,1732441403295 record at close sequenceid=12 2024-11-24T09:43:39,516 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,517 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f1ee6750d931ca2321055907c01bc718, regionState=CLOSED 2024-11-24T09:43:39,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,42075,1732441403106 because future has completed 2024-11-24T09:43:39,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T09:43:39,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,42075,1732441403106 in 192 msec 2024-11-24T09:43:39,525 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, REOPEN/MOVE; state=CLOSED, location=ba35ada4783f,46517,1732441403295; forceNewPlan=false, retain=false 2024-11-24T09:43:39,676 INFO [ba35ada4783f:39325 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T09:43:39,676 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f1ee6750d931ca2321055907c01bc718, regionState=OPENING, regionLocation=ba35ada4783f,46517,1732441403295 2024-11-24T09:43:39,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, REOPEN/MOVE because future has completed 2024-11-24T09:43:39,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,46517,1732441403295}] 2024-11-24T09:43:39,846 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,847 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => f1ee6750d931ca2321055907c01bc718, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:39,847 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,847 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:39,847 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,848 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,850 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,851 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1ee6750d931ca2321055907c01bc718 columnFamilyName cf1 2024-11-24T09:43:39,851 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:39,859 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/b84e24fb63bf40a0bcd4d34c2b222161 2024-11-24T09:43:39,859 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(327): Store=f1ee6750d931ca2321055907c01bc718/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:39,860 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,861 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1ee6750d931ca2321055907c01bc718 columnFamilyName cf2 2024-11-24T09:43:39,861 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:39,867 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf2/8fb57b3ebfbf4b3e8da63bf469eb3c14 2024-11-24T09:43:39,867 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(327): Store=f1ee6750d931ca2321055907c01bc718/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:39,868 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,868 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,870 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,870 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,870 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,871 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T09:43:39,873 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,874 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened f1ee6750d931ca2321055907c01bc718; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67071394, jitterRate=-5.583465099334717E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T09:43:39,874 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,875 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for f1ee6750d931ca2321055907c01bc718: Running coprocessor pre-open hook at 1732441419848Writing region info on filesystem at 1732441419848Initializing all the Stores at 1732441419849 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441419849Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441419850 (+1 ms)Cleaning up temporary data from old regions at 1732441419870 (+20 ms)Running coprocessor post-open hooks at 1732441419874 (+4 ms)Region opened successfully at 1732441419875 (+1 ms) 2024-11-24T09:43:39,876 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., pid=12, masterSystemTime=1732441419838 2024-11-24T09:43:39,878 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,878 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,879 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f1ee6750d931ca2321055907c01bc718, regionState=OPEN, openSeqNum=18, regionLocation=ba35ada4783f,46517,1732441403295 2024-11-24T09:43:39,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,46517,1732441403295 because future has completed 2024-11-24T09:43:39,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-24T09:43:39,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,46517,1732441403295 in 199 msec 2024-11-24T09:43:39,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, REOPEN/MOVE in 565 msec 2024-11-24T09:43:39,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T09:43:39,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T09:43:39,929 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server ba35ada4783f,46517,1732441403295: testing ***** 2024-11-24T09:43:39,929 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T09:43:39,931 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T09:43:39,933 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T09:43:39,937 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T09:43:39,938 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T09:43:39,947 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 276585200 }, "NonHeapMemoryUsage": { "committed": 170655744, "init": 7667712, "max": -1, "used": 168118736 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "ba35ada4783f", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2068, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 9, "ProcessCallTime_98th_percentile": 9, "ProcessCallTime_99th_percentile": 9, "ProcessCallTime_99.9th_percentile": 9, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 10, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "ba35ada4783f", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57552", "tag.serverName": "ba35ada4783f,46517,1732441403295", "tag.clusterId": "574c19c5-3bf5-487a-a197-015a772295c0", "tag.Context": "regionserver", "tag.Hostname": "ba35ada4783f", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732441403295, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 7, "Put_mean": 4, "Put_25th_percentile": 3, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T09:43:39,951 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39325 {}] master.MasterRpcServices(700): ba35ada4783f,46517,1732441403295 reported a fatal error: ***** ABORTING region server ba35ada4783f,46517,1732441403295: testing ***** 2024-11-24T09:43:39,953 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba35ada4783f,46517,1732441403295' ***** 2024-11-24T09:43:39,953 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-24T09:43:39,953 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T09:43:39,954 INFO [RS:2;ba35ada4783f:46517 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T09:43:39,954 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T09:43:39,954 INFO [RS:2;ba35ada4783f:46517 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T09:43:39,954 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(3091): Received CLOSE for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:39,954 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(956): aborting server ba35ada4783f,46517,1732441403295 2024-11-24T09:43:39,954 INFO [RS:2;ba35ada4783f:46517 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T09:43:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:34436 deadline: 1732441479954, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=46517 startCode=1732441403295. As of locationSeqNum=12. 2024-11-24T09:43:39,954 INFO [RS:2;ba35ada4783f:46517 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ba35ada4783f:46517. 2024-11-24T09:43:39,954 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f1ee6750d931ca2321055907c01bc718, disabling compactions & flushes 2024-11-24T09:43:39,955 DEBUG [RS:2;ba35ada4783f:46517 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:43:39,954 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,955 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,955 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. after waiting 0 ms 2024-11-24T09:43:39,955 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,955 DEBUG [RS:2;ba35ada4783f:46517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:43:39,955 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,42075,1732441403106, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,42075,1732441403106, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=46517 startCode=1732441403295. As of locationSeqNum=12. 2024-11-24T09:43:39,955 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,42075,1732441403106, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=46517 startCode=1732441403295. As of locationSeqNum=12. 2024-11-24T09:43:39,955 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,42075,1732441403106, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=46517 startCode=1732441403295. As of locationSeqNum=12. 2024-11-24T09:43:39,955 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T09:43:39,955 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T09:43:39,955 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T09:43:39,955 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T09:43:39,956 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T09:43:39,956 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T09:43:39,956 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,956 DEBUG [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(1325): Online Regions={f1ee6750d931ca2321055907c01bc718=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., 1588230740=hbase:meta,,1.1588230740} 2024-11-24T09:43:39,956 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T09:43:39,956 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f1ee6750d931ca2321055907c01bc718: Waiting for close lock at 1732441419954Running coprocessor pre-close hooks at 1732441419954Disabling compacts and flushes for region at 1732441419954Disabling writes for close at 1732441419955 (+1 ms)Writing region close event to WAL at 1732441419956 (+1 ms)Running coprocessor post-close hooks at 1732441419956Closed at 1732441419956 2024-11-24T09:43:39,957 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T09:43:39,957 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T09:43:39,957 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T09:43:39,957 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:39,957 DEBUG [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T09:43:39,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ba35ada4783f,46517,1732441403295 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:39,957 ERROR [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-24T09:43:39,957 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T09:43:39,957 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T09:43:39,958 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732441419956Running coprocessor pre-close hooks at 1732441419956Disabling compacts and flushes for region at 1732441419956Disabling writes for close at 1732441419957 (+1 ms)Writing region close event to WAL at 1732441419957Running coprocessor post-close hooks at 1732441419957Closed at 1732441419957 2024-11-24T09:43:39,958 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T09:43:39,958 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ba35ada4783f,46517,1732441403295 aborting 2024-11-24T09:43:39,958 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ba35ada4783f,46517,1732441403295 aborting 2024-11-24T09:43:39,958 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 from cache 2024-11-24T09:43:39,959 INFO [regionserver/ba35ada4783f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:43:40,068 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:43:40,070 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1] 2024-11-24T09:43:40,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ba35ada4783f,46517,1732441403295 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:40,071 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ba35ada4783f,46517,1732441403295 aborting 2024-11-24T09:43:40,071 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ba35ada4783f,46517,1732441403295 aborting 2024-11-24T09:43:40,071 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 from cache 2024-11-24T09:43:40,157 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(976): stopping server ba35ada4783f,46517,1732441403295; all regions closed. 2024-11-24T09:43:40,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741836_1012 (size=3561) 2024-11-24T09:43:40,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741836_1012 (size=3561) 2024-11-24T09:43:40,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741836_1012 (size=3561) 2024-11-24T09:43:40,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741835_1011 (size=1403) 2024-11-24T09:43:40,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741835_1011 (size=1403) 2024-11-24T09:43:40,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741835_1011 (size=1403) 2024-11-24T09:43:40,166 DEBUG [RS:2;ba35ada4783f:46517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:43:40,167 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:43:40,167 INFO [RS:2;ba35ada4783f:46517 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T09:43:40,167 INFO [RS:2;ba35ada4783f:46517 {}] hbase.ChoreService(370): Chore service for: regionserver/ba35ada4783f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T09:43:40,167 INFO [RS:2;ba35ada4783f:46517 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T09:43:40,168 INFO [regionserver/ba35ada4783f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T09:43:40,168 INFO [RS:2;ba35ada4783f:46517 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46517 2024-11-24T09:43:40,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T09:43:40,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba35ada4783f,46517,1732441403295 2024-11-24T09:43:40,225 INFO [RS:2;ba35ada4783f:46517 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T09:43:40,227 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba35ada4783f,46517,1732441403295] 2024-11-24T09:43:40,242 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba35ada4783f,46517,1732441403295 already deleted, retry=false 2024-11-24T09:43:40,242 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ba35ada4783f,46517,1732441403295 on ba35ada4783f,39325,1732441402377 2024-11-24T09:43:40,251 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ba35ada4783f,46517,1732441403295, splitWal=true, meta=true 2024-11-24T09:43:40,254 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for ba35ada4783f,46517,1732441403295 (carryingMeta=true) ba35ada4783f,46517,1732441403295/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2a7ee93b[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T09:43:40,254 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ba35ada4783f,46517,1732441403295, splitWal=true, meta=true 2024-11-24T09:43:40,255 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ba35ada4783f,46517,1732441403295, splitWal=true, meta=true, isMeta: true 2024-11-24T09:43:40,257 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting 2024-11-24T09:43:40,258 INFO [PEWorker-2 {}] master.SplitWALManager(105): ba35ada4783f,46517,1732441403295 WAL count=1, meta=true 2024-11-24T09:43:40,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta}] 2024-11-24T09:43:40,266 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=ba35ada4783f,42075,1732441403106 2024-11-24T09:43:40,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta, worker=ba35ada4783f,42075,1732441403106}] 2024-11-24T09:43:40,277 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:43:40,279 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1] 2024-11-24T09:43:40,280 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ba35ada4783f:46517 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ba35ada4783f/172.17.0.2:46517 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:40,281 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1, error=java.net.ConnectException: Call to address=ba35ada4783f:46517 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ba35ada4783f/172.17.0.2:46517 2024-11-24T09:43:40,281 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-24T09:43:40,281 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 from cache 2024-11-24T09:43:40,281 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address ba35ada4783f:46517 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ba35ada4783f/172.17.0.2:46517 2024-11-24T09:43:40,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:43:40,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46517-0x1016d837bd10003, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:43:40,334 INFO [RS:2;ba35ada4783f:46517 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T09:43:40,334 INFO [RS:2;ba35ada4783f:46517 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba35ada4783f,46517,1732441403295; zookeeper connection closed. 2024-11-24T09:43:40,335 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2a098c95 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2a098c95 2024-11-24T09:43:40,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42075 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-24T09:43:40,449 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta, size=3.5 K (3561bytes) 2024-11-24T09:43:40,449 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta 2024-11-24T09:43:40,450 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta after 1ms 2024-11-24T09:43:40,452 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:40,453 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta took 4ms 2024-11-24T09:43:40,460 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-24T09:43:40,461 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta so closing down 2024-11-24T09:43:40,461 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:40,462 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta.temp 2024-11-24T09:43:40,463 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta.temp 2024-11-24T09:43:40,464 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:40,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741892_1070 (size=3346) 2024-11-24T09:43:40,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741892_1070 (size=3346) 2024-11-24T09:43:40,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741892_1070 (size=3346) 2024-11-24T09:43:40,478 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T09:43:40,479 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta.temp to hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T09:43:40,479 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 26 ms; skipped=1; WAL=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta, size=3.5 K, length=3561, corrupted=false, cancelled=false 2024-11-24T09:43:40,479 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta, journal: Splitting hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta, size=3.5 K (3561bytes) at 1732441420449Finishing writing output for hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta so closing down at 1732441420461 (+12 ms)Creating recovered edits writer path=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta.temp at 1732441420463 (+2 ms)3 split writer threads finished at 1732441420464 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732441420478 (+14 ms)Rename recovered edits hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta.temp to hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732441420479 (+1 ms)Processed 16 edits across 1 Regions in 26 ms; skipped=1; WAL=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta, size=3.5 K, length=3561, corrupted=false, cancelled=false at 1732441420479 2024-11-24T09:43:40,480 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta 2024-11-24T09:43:40,481 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-24T09:43:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-24T09:43:40,488 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta to hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/oldWALs 2024-11-24T09:43:40,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T09:43:40,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta, worker=ba35ada4783f,42075,1732441403106 in 220 msec 2024-11-24T09:43:40,492 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=ba35ada4783f,42075,1732441403106 2024-11-24T09:43:40,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T09:43:40,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure ba35ada4783f%2C46517%2C1732441403295.meta.1732441405249.meta, worker=ba35ada4783f,42075,1732441403106 in 234 msec 2024-11-24T09:43:40,498 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting dir is empty, no logs to split. 2024-11-24T09:43:40,498 INFO [PEWorker-2 {}] master.SplitWALManager(105): ba35ada4783f,46517,1732441403295 WAL count=0, meta=true 2024-11-24T09:43:40,498 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if ba35ada4783f,46517,1732441403295 WAL splitting is done? wals=0, meta=true 2024-11-24T09:43:40,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T09:43:40,501 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T09:43:40,502 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T09:43:40,588 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:43:40,590 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1] 2024-11-24T09:43:40,590 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to ba35ada4783f:46517 this server is in the failed servers list 2024-11-24T09:43:40,591 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ba35ada4783f:46517 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ba35ada4783f:46517 2024-11-24T09:43:40,591 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ba35ada4783f:46517 2024-11-24T09:43:40,591 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 from cache 2024-11-24T09:43:40,652 DEBUG [ba35ada4783f:39325 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T09:43:40,653 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(204): Hosts are {ba35ada4783f=0} racks are {/default-rack=0} 2024-11-24T09:43:40,653 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T09:43:40,653 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T09:43:40,653 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T09:43:40,653 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T09:43:40,653 INFO [ba35ada4783f:39325 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T09:43:40,653 INFO [ba35ada4783f:39325 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T09:43:40,653 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T09:43:40,654 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba35ada4783f,42075,1732441403106 2024-11-24T09:43:40,655 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba35ada4783f,42075,1732441403106, state=OPENING 2024-11-24T09:43:40,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:43:40,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:43:40,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:43:40,692 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T09:43:40,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:40,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:40,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:40,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba35ada4783f,42075,1732441403106}] 2024-11-24T09:43:40,856 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T09:43:40,856 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:40,856 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T09:43:40,859 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C42075%2C1732441403106.meta, suffix=.meta, logDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,42075,1732441403106, archiveDir=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/oldWALs, maxLogs=32 2024-11-24T09:43:40,874 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,42075,1732441403106/ba35ada4783f%2C42075%2C1732441403106.meta.1732441420860.meta, exclude list is [], retry=0 2024-11-24T09:43:40,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:40,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:40,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:40,879 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,42075,1732441403106/ba35ada4783f%2C42075%2C1732441403106.meta.1732441420860.meta 2024-11-24T09:43:40,880 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:35185:35185)] 2024-11-24T09:43:40,880 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:40,881 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T09:43:40,881 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T09:43:40,881 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T09:43:40,881 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T09:43:40,881 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:40,881 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T09:43:40,881 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T09:43:40,883 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T09:43:40,883 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T09:43:40,884 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:40,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:40,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T09:43:40,885 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T09:43:40,885 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:40,886 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:40,886 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T09:43:40,886 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T09:43:40,886 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:40,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:40,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T09:43:40,888 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T09:43:40,888 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:40,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:43:40,889 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T09:43:40,890 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740 2024-11-24T09:43:40,891 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740 2024-11-24T09:43:40,892 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T09:43:40,894 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/0000000000000000018: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:40,896 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T09:43:40,896 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-24T09:43:40,915 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/.tmp/info/10e18d12da5f440e8c94e29336345cfc is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718./info:regioninfo/1732441419879/Put/seqid=0 2024-11-24T09:43:40,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741894_1072 (size=11177) 2024-11-24T09:43:40,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741894_1072 (size=11177) 2024-11-24T09:43:40,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741894_1072 (size=11177) 2024-11-24T09:43:40,922 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/.tmp/info/10e18d12da5f440e8c94e29336345cfc 2024-11-24T09:43:40,942 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/.tmp/ns/e075fa9db6aa48358c07e417a5f7a38f is 43, key is default/ns:d/1732441405437/Put/seqid=0 2024-11-24T09:43:40,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741895_1073 (size=5153) 2024-11-24T09:43:40,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741895_1073 (size=5153) 2024-11-24T09:43:40,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741895_1073 (size=5153) 2024-11-24T09:43:40,949 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/.tmp/ns/e075fa9db6aa48358c07e417a5f7a38f 2024-11-24T09:43:40,967 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/.tmp/table/871e8e822fed41d8bab24031785acf68 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732441418052/Put/seqid=0 2024-11-24T09:43:40,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741896_1074 (size=5431) 2024-11-24T09:43:40,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741896_1074 (size=5431) 2024-11-24T09:43:40,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741896_1074 (size=5431) 2024-11-24T09:43:40,974 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/.tmp/table/871e8e822fed41d8bab24031785acf68 2024-11-24T09:43:40,981 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/.tmp/info/10e18d12da5f440e8c94e29336345cfc as hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/info/10e18d12da5f440e8c94e29336345cfc 2024-11-24T09:43:40,989 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/info/10e18d12da5f440e8c94e29336345cfc, entries=36, sequenceid=18, filesize=10.9 K 2024-11-24T09:43:40,990 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/.tmp/ns/e075fa9db6aa48358c07e417a5f7a38f as hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/ns/e075fa9db6aa48358c07e417a5f7a38f 2024-11-24T09:43:40,997 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/ns/e075fa9db6aa48358c07e417a5f7a38f, entries=2, sequenceid=18, filesize=5.0 K 2024-11-24T09:43:40,999 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/.tmp/table/871e8e822fed41d8bab24031785acf68 as hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/table/871e8e822fed41d8bab24031785acf68 2024-11-24T09:43:41,006 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/table/871e8e822fed41d8bab24031785acf68, entries=2, sequenceid=18, filesize=5.3 K 2024-11-24T09:43:41,007 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=18, compaction requested=false; wal=null 2024-11-24T09:43:41,008 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T09:43:41,010 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T09:43:41,010 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T09:43:41,011 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T09:43:41,012 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T09:43:41,015 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-24T09:43:41,016 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65567587, jitterRate=-0.022966817021369934}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T09:43:41,016 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T09:43:41,016 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732441420881Writing region info on filesystem at 1732441420881Initializing all the Stores at 1732441420882 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441420882Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441420882Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441420882Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441420883 (+1 ms)Obtaining lock to block concurrent updates at 1732441420896 (+13 ms)Preparing flush snapshotting stores in 1588230740 at 1732441420896Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732441420897 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732441420897Flushing 1588230740/info: creating writer at 1732441420897Flushing 1588230740/info: appending metadata at 1732441420915 (+18 ms)Flushing 1588230740/info: closing flushed file at 1732441420915Flushing 1588230740/ns: creating writer at 1732441420928 (+13 ms)Flushing 1588230740/ns: appending metadata at 1732441420941 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732441420942 (+1 ms)Flushing 1588230740/table: creating writer at 1732441420954 (+12 ms)Flushing 1588230740/table: appending metadata at 1732441420967 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732441420967Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c2e1cca: reopening flushed file at 1732441420980 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36304623: reopening flushed file at 1732441420989 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74569e20: reopening flushed file at 1732441420998 (+9 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=18, compaction requested=false; wal=null at 1732441421007 (+9 ms)Cleaning up temporary data from old regions at 1732441421010 (+3 ms)Running coprocessor post-open hooks at 1732441421016 (+6 ms)Region opened successfully at 1732441421016 2024-11-24T09:43:41,018 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732441420849 2024-11-24T09:43:41,020 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T09:43:41,020 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T09:43:41,021 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=ba35ada4783f,42075,1732441403106 2024-11-24T09:43:41,022 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba35ada4783f,42075,1732441403106, state=OPEN 2024-11-24T09:43:41,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:43:41,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:43:41,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:43:41,050 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba35ada4783f,42075,1732441403106 2024-11-24T09:43:41,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:41,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:41,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:43:41,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-24T09:43:41,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba35ada4783f,42075,1732441403106 in 358 msec 2024-11-24T09:43:41,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-24T09:43:41,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 553 msec 2024-11-24T09:43:41,057 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): ba35ada4783f,46517,1732441403295 had 2 regions 2024-11-24T09:43:41,058 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ba35ada4783f,46517,1732441403295, splitWal=true, meta=true, isMeta: false 2024-11-24T09:43:41,060 INFO [PEWorker-1 {}] master.SplitWALManager(105): ba35ada4783f,46517,1732441403295 WAL count=1, meta=false 2024-11-24T09:43:41,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ba35ada4783f%2C46517%2C1732441403295.1732441404802}] 2024-11-24T09:43:41,062 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=ba35ada4783f,42075,1732441403106 2024-11-24T09:43:41,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ba35ada4783f%2C46517%2C1732441403295.1732441404802, worker=ba35ada4783f,42075,1732441403106}] 2024-11-24T09:43:41,098 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:43:41,100 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42075,1732441403106, seqNum=-1] 2024-11-24T09:43:41,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42075 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-24T09:43:41,218 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ba35ada4783f:46517 this server is in the failed servers list 2024-11-24T09:43:41,219 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ba35ada4783f:46517 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ba35ada4783f:46517 2024-11-24T09:43:41,219 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ba35ada4783f:46517 2024-11-24T09:43:41,219 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=12 from cache 2024-11-24T09:43:41,232 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802, size=1.4 K (1403bytes) 2024-11-24T09:43:41,232 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802 2024-11-24T09:43:41,233 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802 after 0ms 2024-11-24T09:43:41,236 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:41,236 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802 took 4ms 2024-11-24T09:43:41,239 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for f1ee6750d931ca2321055907c01bc718: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-24T09:43:41,239 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802 so closing down 2024-11-24T09:43:41,239 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:41,239 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:41,239 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802, size=1.4 K, length=1403, corrupted=false, cancelled=false 2024-11-24T09:43:41,239 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802, journal: Splitting hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802, size=1.4 K (1403bytes) at 1732441421232Finishing writing output for hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802 so closing down at 1732441421239 (+7 ms)3 split writer threads finished at 1732441421239Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802, size=1.4 K, length=1403, corrupted=false, cancelled=false at 1732441421239 2024-11-24T09:43:41,239 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802 2024-11-24T09:43:41,239 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-24T09:43:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39325 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-24T09:43:41,244 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting/ba35ada4783f%2C46517%2C1732441403295.1732441404802 to hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/oldWALs 2024-11-24T09:43:41,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-24T09:43:41,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba35ada4783f%2C46517%2C1732441403295.1732441404802, worker=ba35ada4783f,42075,1732441403106 in 181 msec 2024-11-24T09:43:41,248 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=ba35ada4783f,42075,1732441403106 2024-11-24T09:43:41,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-24T09:43:41,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure ba35ada4783f%2C46517%2C1732441403295.1732441404802, worker=ba35ada4783f,42075,1732441403106 in 188 msec 2024-11-24T09:43:41,253 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/WALs/ba35ada4783f,46517,1732441403295-splitting dir is empty, no logs to split. 2024-11-24T09:43:41,253 INFO [PEWorker-1 {}] master.SplitWALManager(105): ba35ada4783f,46517,1732441403295 WAL count=0, meta=false 2024-11-24T09:43:41,253 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ba35ada4783f,46517,1732441403295 WAL splitting is done? wals=0, meta=false 2024-11-24T09:43:41,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, ASSIGN}] 2024-11-24T09:43:41,257 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, ASSIGN 2024-11-24T09:43:41,258 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T09:43:41,409 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(204): Hosts are {ba35ada4783f=0} racks are {/default-rack=0} 2024-11-24T09:43:41,409 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T09:43:41,409 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T09:43:41,409 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T09:43:41,409 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T09:43:41,409 INFO [ba35ada4783f:39325 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T09:43:41,409 INFO [ba35ada4783f:39325 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T09:43:41,409 DEBUG [ba35ada4783f:39325 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T09:43:41,410 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f1ee6750d931ca2321055907c01bc718, regionState=OPENING, regionLocation=ba35ada4783f,34867,1732441403238 2024-11-24T09:43:41,412 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ba35ada4783f:46517 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba35ada4783f/172.17.0.2:46517 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:41,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1, error=java.net.ConnectException: Call to address=ba35ada4783f:46517 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba35ada4783f/172.17.0.2:46517 2024-11-24T09:43:41,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-24T09:43:41,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,46517,1732441403295, seqNum=-1 from cache 2024-11-24T09:43:41,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address ba35ada4783f:46517 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba35ada4783f/172.17.0.2:46517 2024-11-24T09:43:41,434 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=18] 2024-11-24T09:43:41,434 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to ba35ada4783f:46517 this server is in the failed servers list 2024-11-24T09:43:41,435 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ba35ada4783f:46517 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ba35ada4783f:46517 2024-11-24T09:43:41,435 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ba35ada4783f:46517 2024-11-24T09:43:41,435 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,46517,1732441403295, seqNum=18 from cache 2024-11-24T09:43:41,518 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:43:41,518 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42075,1732441403106, seqNum=-1] 2024-11-24T09:43:41,519 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T09:43:41,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53707, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T09:43:41,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, ASSIGN because future has completed 2024-11-24T09:43:41,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,34867,1732441403238}] 2024-11-24T09:43:41,680 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T09:43:41,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53649, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T09:43:41,693 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:41,694 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => f1ee6750d931ca2321055907c01bc718, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:41,694 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:41,694 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:41,694 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:41,694 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:41,696 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:41,698 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1ee6750d931ca2321055907c01bc718 columnFamilyName cf1 2024-11-24T09:43:41,698 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:41,707 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf1/b84e24fb63bf40a0bcd4d34c2b222161 2024-11-24T09:43:41,707 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(327): Store=f1ee6750d931ca2321055907c01bc718/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:41,708 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:41,709 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1ee6750d931ca2321055907c01bc718 columnFamilyName cf2 2024-11-24T09:43:41,709 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:41,715 DEBUG [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/cf2/8fb57b3ebfbf4b3e8da63bf469eb3c14 2024-11-24T09:43:41,715 INFO [StoreOpener-f1ee6750d931ca2321055907c01bc718-1 {}] regionserver.HStore(327): Store=f1ee6750d931ca2321055907c01bc718/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:41,715 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:41,716 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:41,717 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:41,718 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:41,718 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:41,719 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T09:43:41,720 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:41,721 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened f1ee6750d931ca2321055907c01bc718; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66323769, jitterRate=-0.011698827147483826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T09:43:41,721 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:43:41,722 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for f1ee6750d931ca2321055907c01bc718: Running coprocessor pre-open hook at 1732441421695Writing region info on filesystem at 1732441421695Initializing all the Stores at 1732441421696 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441421696Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441421696Cleaning up temporary data from old regions at 1732441421718 (+22 ms)Running coprocessor post-open hooks at 1732441421721 (+3 ms)Region opened successfully at 1732441421722 (+1 ms) 2024-11-24T09:43:41,723 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., pid=21, masterSystemTime=1732441421679 2024-11-24T09:43:41,726 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:41,726 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:43:41,727 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f1ee6750d931ca2321055907c01bc718, regionState=OPEN, openSeqNum=18, regionLocation=ba35ada4783f,34867,1732441403238 2024-11-24T09:43:41,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,34867,1732441403238 because future has completed 2024-11-24T09:43:41,733 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-24T09:43:41,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure f1ee6750d931ca2321055907c01bc718, server=ba35ada4783f,34867,1732441403238 in 205 msec 2024-11-24T09:43:41,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-24T09:43:41,737 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server ba35ada4783f,46517,1732441403295 after splitting done 2024-11-24T09:43:41,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f1ee6750d931ca2321055907c01bc718, ASSIGN in 478 msec 2024-11-24T09:43:41,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure ba35ada4783f,46517,1732441403295, splitWal=true, meta=true in 1.4940 sec 2024-11-24T09:43:41,743 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718., hostname=ba35ada4783f,34867,1732441403238, seqNum=18] 2024-11-24T09:43:41,743 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T09:43:41,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T09:43:41,762 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=407 (was 406) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741893_1071, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-549267916_22 at /127.0.0.1:37328 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-549267916_22 at /127.0.0.1:40746 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741893_1071] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-549267916_22 at /127.0.0.1:45766 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741893_1071] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-416840634_22 at /127.0.0.1:40646 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741893_1071, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-549267916_22 at /127.0.0.1:37416 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741893_1071] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741893_1071, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-416840634_22 at /127.0.0.1:45702 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88-prefix:ba35ada4783f,42075,1732441403106.meta java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1050 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=283 (was 308), ProcessCount=11 (was 11), AvailableMemoryMB=7632 (was 7670) 2024-11-24T09:43:41,764 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-24T09:43:41,777 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=407, OpenFileDescriptor=1050, MaxFileDescriptor=1048576, SystemLoadAverage=283, ProcessCount=11, AvailableMemoryMB=7631 2024-11-24T09:43:41,777 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-24T09:43:41,792 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:41,795 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:41,796 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:41,799 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-56844351, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/hregion-56844351, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:41,811 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-56844351/hregion-56844351.1732441421799, exclude list is [], retry=0 2024-11-24T09:43:41,814 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:41,814 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:41,814 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:41,816 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-56844351/hregion-56844351.1732441421799 2024-11-24T09:43:41,817 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:35185:35185)] 2024-11-24T09:43:41,817 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3836c53143d9d39f89238a2ef580d0f0, NAME => 'testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36549/hbase 2024-11-24T09:43:41,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741898_1076 (size=67) 2024-11-24T09:43:41,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741898_1076 (size=67) 2024-11-24T09:43:41,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741898_1076 (size=67) 2024-11-24T09:43:41,827 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:41,831 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,832 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3836c53143d9d39f89238a2ef580d0f0 columnFamilyName a 2024-11-24T09:43:41,832 DEBUG [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:41,833 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(327): Store=3836c53143d9d39f89238a2ef580d0f0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:41,833 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,834 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3836c53143d9d39f89238a2ef580d0f0 columnFamilyName b 2024-11-24T09:43:41,834 DEBUG [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:41,835 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(327): Store=3836c53143d9d39f89238a2ef580d0f0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:41,835 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,836 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3836c53143d9d39f89238a2ef580d0f0 columnFamilyName c 2024-11-24T09:43:41,836 DEBUG [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:41,837 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(327): Store=3836c53143d9d39f89238a2ef580d0f0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:41,837 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,837 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,838 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,839 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,839 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,839 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:41,840 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,842 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:41,843 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3836c53143d9d39f89238a2ef580d0f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67481175, jitterRate=0.005547866225242615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:41,844 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3836c53143d9d39f89238a2ef580d0f0: Writing region info on filesystem at 1732441421827Initializing all the Stores at 1732441421830 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441421830Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441421830Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441421831 (+1 ms)Cleaning up temporary data from old regions at 1732441421839 (+8 ms)Region opened successfully at 1732441421843 (+4 ms) 2024-11-24T09:43:41,844 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3836c53143d9d39f89238a2ef580d0f0, disabling compactions & flushes 2024-11-24T09:43:41,844 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0. 2024-11-24T09:43:41,844 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0. 2024-11-24T09:43:41,844 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0. after waiting 0 ms 2024-11-24T09:43:41,844 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0. 2024-11-24T09:43:41,844 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0. 2024-11-24T09:43:41,844 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3836c53143d9d39f89238a2ef580d0f0: Waiting for close lock at 1732441421844Disabling compacts and flushes for region at 1732441421844Disabling writes for close at 1732441421844Writing region close event to WAL at 1732441421844Closed at 1732441421844 2024-11-24T09:43:41,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741897_1075 (size=95) 2024-11-24T09:43:41,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741897_1075 (size=95) 2024-11-24T09:43:41,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741897_1075 (size=95) 2024-11-24T09:43:41,849 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:41,850 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-56844351:(num 1732441421799) 2024-11-24T09:43:41,850 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:41,852 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:41,867 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852, exclude list is [], retry=0 2024-11-24T09:43:41,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:41,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:41,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:41,872 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852 2024-11-24T09:43:41,873 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591)] 2024-11-24T09:43:41,873 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3836c53143d9d39f89238a2ef580d0f0, NAME => 'testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:41,874 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:41,874 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,874 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,875 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,876 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3836c53143d9d39f89238a2ef580d0f0 columnFamilyName a 2024-11-24T09:43:41,876 DEBUG [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:41,876 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(327): Store=3836c53143d9d39f89238a2ef580d0f0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:41,876 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,877 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3836c53143d9d39f89238a2ef580d0f0 columnFamilyName b 2024-11-24T09:43:41,877 DEBUG [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:41,878 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(327): Store=3836c53143d9d39f89238a2ef580d0f0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:41,878 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,878 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3836c53143d9d39f89238a2ef580d0f0 columnFamilyName c 2024-11-24T09:43:41,878 DEBUG [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:41,879 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(327): Store=3836c53143d9d39f89238a2ef580d0f0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:41,879 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,880 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,881 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,882 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,882 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,883 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:41,884 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:41,885 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3836c53143d9d39f89238a2ef580d0f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75008284, jitterRate=0.11771053075790405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:41,886 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3836c53143d9d39f89238a2ef580d0f0: Writing region info on filesystem at 1732441421874Initializing all the Stores at 1732441421875 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441421875Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441421875Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441421875Cleaning up temporary data from old regions at 1732441421882 (+7 ms)Region opened successfully at 1732441421886 (+4 ms) 2024-11-24T09:43:41,913 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3836c53143d9d39f89238a2ef580d0f0 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T09:43:41,936 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/.tmp/a/23aa4f7ec78d4b1980d6febf23560ecf is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732441421886/Put/seqid=0 2024-11-24T09:43:41,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741900_1078 (size=5958) 2024-11-24T09:43:41,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741900_1078 (size=5958) 2024-11-24T09:43:41,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741900_1078 (size=5958) 2024-11-24T09:43:41,944 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/.tmp/a/23aa4f7ec78d4b1980d6febf23560ecf 2024-11-24T09:43:41,972 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/.tmp/b/f3aab04749704d62a527643868e295bf is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732441421896/Put/seqid=0 2024-11-24T09:43:41,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741901_1079 (size=5958) 2024-11-24T09:43:41,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741901_1079 (size=5958) 2024-11-24T09:43:41,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741901_1079 (size=5958) 2024-11-24T09:43:41,983 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/.tmp/b/f3aab04749704d62a527643868e295bf 2024-11-24T09:43:42,001 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/.tmp/c/7075013d9c0b4fee8a98e2eff8987bee is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732441421904/Put/seqid=0 2024-11-24T09:43:42,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741902_1080 (size=5958) 2024-11-24T09:43:42,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741902_1080 (size=5958) 2024-11-24T09:43:42,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741902_1080 (size=5958) 2024-11-24T09:43:42,009 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/.tmp/c/7075013d9c0b4fee8a98e2eff8987bee 2024-11-24T09:43:42,015 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/.tmp/a/23aa4f7ec78d4b1980d6febf23560ecf as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/a/23aa4f7ec78d4b1980d6febf23560ecf 2024-11-24T09:43:42,021 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/a/23aa4f7ec78d4b1980d6febf23560ecf, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T09:43:42,022 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/.tmp/b/f3aab04749704d62a527643868e295bf as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/b/f3aab04749704d62a527643868e295bf 2024-11-24T09:43:42,028 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/b/f3aab04749704d62a527643868e295bf, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T09:43:42,029 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/.tmp/c/7075013d9c0b4fee8a98e2eff8987bee as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/c/7075013d9c0b4fee8a98e2eff8987bee 2024-11-24T09:43:42,035 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/c/7075013d9c0b4fee8a98e2eff8987bee, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T09:43:42,037 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 3836c53143d9d39f89238a2ef580d0f0 in 124ms, sequenceid=33, compaction requested=false 2024-11-24T09:43:42,037 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3836c53143d9d39f89238a2ef580d0f0: 2024-11-24T09:43:42,037 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3836c53143d9d39f89238a2ef580d0f0, disabling compactions & flushes 2024-11-24T09:43:42,037 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0. 2024-11-24T09:43:42,037 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0. 2024-11-24T09:43:42,037 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0. after waiting 0 ms 2024-11-24T09:43:42,037 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0. 2024-11-24T09:43:42,038 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0. 2024-11-24T09:43:42,039 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3836c53143d9d39f89238a2ef580d0f0: Waiting for close lock at 1732441422037Disabling compacts and flushes for region at 1732441422037Disabling writes for close at 1732441422037Writing region close event to WAL at 1732441422038 (+1 ms)Closed at 1732441422038 2024-11-24T09:43:42,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741899_1077 (size=3386) 2024-11-24T09:43:42,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741899_1077 (size=3386) 2024-11-24T09:43:42,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741899_1077 (size=3386) 2024-11-24T09:43:42,045 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/b/f3aab04749704d62a527643868e295bf to hdfs://localhost:36549/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/b/f3aab04749704d62a527643868e295bf 2024-11-24T09:43:42,061 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852, size=3.3 K (3386bytes) 2024-11-24T09:43:42,061 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852 2024-11-24T09:43:42,062 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852 after 1ms 2024-11-24T09:43:42,064 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:42,064 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852 took 3ms 2024-11-24T09:43:42,067 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852 so closing down 2024-11-24T09:43:42,067 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:42,067 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732441421852.temp 2024-11-24T09:43:42,069 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000003-wal.1732441421852.temp 2024-11-24T09:43:42,069 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:42,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741903_1081 (size=2944) 2024-11-24T09:43:42,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741903_1081 (size=2944) 2024-11-24T09:43:42,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741903_1081 (size=2944) 2024-11-24T09:43:42,076 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000003-wal.1732441421852.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T09:43:42,077 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000003-wal.1732441421852.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000032 2024-11-24T09:43:42,077 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852, size=3.3 K, length=3386, corrupted=false, cancelled=false 2024-11-24T09:43:42,077 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852, journal: Splitting hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852, size=3.3 K (3386bytes) at 1732441422061Finishing writing output for hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852 so closing down at 1732441422067 (+6 ms)Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000003-wal.1732441421852.temp at 1732441422069 (+2 ms)3 split writer threads finished at 1732441422069Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000003-wal.1732441421852.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732441422076 (+7 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000003-wal.1732441421852.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000032 at 1732441422077 (+1 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852, size=3.3 K, length=3386, corrupted=false, cancelled=false at 1732441422077 2024-11-24T09:43:42,079 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441421852 to hdfs://localhost:36549/hbase/oldWALs/wal.1732441421852 2024-11-24T09:43:42,080 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000032 2024-11-24T09:43:42,080 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:42,081 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:42,095 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441422082, exclude list is [], retry=0 2024-11-24T09:43:42,097 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:42,098 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:42,098 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:42,101 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441422082 2024-11-24T09:43:42,102 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:42,102 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3836c53143d9d39f89238a2ef580d0f0, NAME => 'testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:42,102 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:42,102 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:42,102 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:42,104 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:42,105 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3836c53143d9d39f89238a2ef580d0f0 columnFamilyName a 2024-11-24T09:43:42,105 DEBUG [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:42,111 DEBUG [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/a/23aa4f7ec78d4b1980d6febf23560ecf 2024-11-24T09:43:42,111 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(327): Store=3836c53143d9d39f89238a2ef580d0f0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:42,111 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:42,113 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3836c53143d9d39f89238a2ef580d0f0 columnFamilyName b 2024-11-24T09:43:42,113 DEBUG [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:42,113 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(327): Store=3836c53143d9d39f89238a2ef580d0f0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:42,114 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:42,115 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3836c53143d9d39f89238a2ef580d0f0 columnFamilyName c 2024-11-24T09:43:42,115 DEBUG [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:42,120 DEBUG [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/c/7075013d9c0b4fee8a98e2eff8987bee 2024-11-24T09:43:42,120 INFO [StoreOpener-3836c53143d9d39f89238a2ef580d0f0-1 {}] regionserver.HStore(327): Store=3836c53143d9d39f89238a2ef580d0f0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:42,120 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:42,121 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:42,123 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:42,124 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000032 2024-11-24T09:43:42,126 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:42,128 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000032 2024-11-24T09:43:42,128 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3836c53143d9d39f89238a2ef580d0f0 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T09:43:42,144 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/.tmp/b/08d0320faa914af1adfc32f0ca59759c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732441421896/Put/seqid=0 2024-11-24T09:43:42,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741905_1083 (size=5958) 2024-11-24T09:43:42,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741905_1083 (size=5958) 2024-11-24T09:43:42,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741905_1083 (size=5958) 2024-11-24T09:43:42,151 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/.tmp/b/08d0320faa914af1adfc32f0ca59759c 2024-11-24T09:43:42,156 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/.tmp/b/08d0320faa914af1adfc32f0ca59759c as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/b/08d0320faa914af1adfc32f0ca59759c 2024-11-24T09:43:42,162 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/b/08d0320faa914af1adfc32f0ca59759c, entries=10, sequenceid=32, filesize=5.8 K 2024-11-24T09:43:42,163 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 3836c53143d9d39f89238a2ef580d0f0 in 35ms, sequenceid=32, compaction requested=false; wal=null 2024-11-24T09:43:42,163 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/0000000000000000032 2024-11-24T09:43:42,165 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:42,165 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:42,165 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:42,166 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3836c53143d9d39f89238a2ef580d0f0 2024-11-24T09:43:42,169 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/3836c53143d9d39f89238a2ef580d0f0/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-24T09:43:42,170 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3836c53143d9d39f89238a2ef580d0f0; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61527303, jitterRate=-0.08317174017429352}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:42,170 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3836c53143d9d39f89238a2ef580d0f0: Writing region info on filesystem at 1732441422102Initializing all the Stores at 1732441422103 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441422103Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441422104 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441422104Obtaining lock to block concurrent updates at 1732441422128 (+24 ms)Preparing flush snapshotting stores in 3836c53143d9d39f89238a2ef580d0f0 at 1732441422128Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732441422129 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1732441421793.3836c53143d9d39f89238a2ef580d0f0. at 1732441422129Flushing 3836c53143d9d39f89238a2ef580d0f0/b: creating writer at 1732441422129Flushing 3836c53143d9d39f89238a2ef580d0f0/b: appending metadata at 1732441422143 (+14 ms)Flushing 3836c53143d9d39f89238a2ef580d0f0/b: closing flushed file at 1732441422143Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6131c97a: reopening flushed file at 1732441422156 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 3836c53143d9d39f89238a2ef580d0f0 in 35ms, sequenceid=32, compaction requested=false; wal=null at 1732441422163 (+7 ms)Cleaning up temporary data from old regions at 1732441422165 (+2 ms)Region opened successfully at 1732441422170 (+5 ms) 2024-11-24T09:43:42,194 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=415 (was 407) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:33546 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:37328 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:40646 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:52762 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:45702 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:40056 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1050) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=283 (was 283), ProcessCount=11 (was 11), AvailableMemoryMB=7612 (was 7631) 2024-11-24T09:43:42,194 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-24T09:43:42,205 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=415, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=283, ProcessCount=11, AvailableMemoryMB=7611 2024-11-24T09:43:42,205 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-24T09:43:42,220 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:42,222 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:42,222 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:42,225 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-12320186, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/hregion-12320186, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:42,239 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-12320186/hregion-12320186.1732441422225, exclude list is [], retry=0 2024-11-24T09:43:42,242 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:42,242 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:42,242 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:42,247 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-12320186/hregion-12320186.1732441422225 2024-11-24T09:43:42,247 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:42,247 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 14d2e44cf0df0f7e799ac7ef3309f539, NAME => 'testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36549/hbase 2024-11-24T09:43:42,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741907_1085 (size=68) 2024-11-24T09:43:42,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741907_1085 (size=68) 2024-11-24T09:43:42,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741907_1085 (size=68) 2024-11-24T09:43:42,263 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:42,264 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,266 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14d2e44cf0df0f7e799ac7ef3309f539 columnFamilyName a 2024-11-24T09:43:42,266 DEBUG [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:42,266 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(327): Store=14d2e44cf0df0f7e799ac7ef3309f539/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:42,266 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,268 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14d2e44cf0df0f7e799ac7ef3309f539 columnFamilyName b 2024-11-24T09:43:42,268 DEBUG [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:42,268 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(327): Store=14d2e44cf0df0f7e799ac7ef3309f539/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:42,269 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,270 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14d2e44cf0df0f7e799ac7ef3309f539 columnFamilyName c 2024-11-24T09:43:42,270 DEBUG [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:42,270 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(327): Store=14d2e44cf0df0f7e799ac7ef3309f539/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:42,271 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,271 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,272 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,273 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,273 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,273 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:42,274 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,276 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:42,277 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 14d2e44cf0df0f7e799ac7ef3309f539; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59325249, jitterRate=-0.11598490178585052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:42,278 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 14d2e44cf0df0f7e799ac7ef3309f539: Writing region info on filesystem at 1732441422263Initializing all the Stores at 1732441422264 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441422264Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441422264Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441422264Cleaning up temporary data from old regions at 1732441422273 (+9 ms)Region opened successfully at 1732441422277 (+4 ms) 2024-11-24T09:43:42,278 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 14d2e44cf0df0f7e799ac7ef3309f539, disabling compactions & flushes 2024-11-24T09:43:42,278 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. 2024-11-24T09:43:42,278 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. 2024-11-24T09:43:42,278 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. after waiting 0 ms 2024-11-24T09:43:42,278 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. 2024-11-24T09:43:42,278 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. 2024-11-24T09:43:42,278 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 14d2e44cf0df0f7e799ac7ef3309f539: Waiting for close lock at 1732441422278Disabling compacts and flushes for region at 1732441422278Disabling writes for close at 1732441422278Writing region close event to WAL at 1732441422278Closed at 1732441422278 2024-11-24T09:43:42,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741906_1084 (size=95) 2024-11-24T09:43:42,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741906_1084 (size=95) 2024-11-24T09:43:42,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741906_1084 (size=95) 2024-11-24T09:43:42,284 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:42,284 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-12320186:(num 1732441422225) 2024-11-24T09:43:42,284 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:42,286 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:42,299 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286, exclude list is [], retry=0 2024-11-24T09:43:42,302 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:42,302 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:42,303 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:42,305 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286 2024-11-24T09:43:42,306 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:42,362 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 14d2e44cf0df0f7e799ac7ef3309f539, NAME => 'testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:42,364 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,364 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:42,364 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,364 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,367 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,368 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14d2e44cf0df0f7e799ac7ef3309f539 columnFamilyName a 2024-11-24T09:43:42,368 DEBUG [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:42,368 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(327): Store=14d2e44cf0df0f7e799ac7ef3309f539/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:42,369 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,369 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14d2e44cf0df0f7e799ac7ef3309f539 columnFamilyName b 2024-11-24T09:43:42,369 DEBUG [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:42,370 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(327): Store=14d2e44cf0df0f7e799ac7ef3309f539/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:42,370 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,371 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14d2e44cf0df0f7e799ac7ef3309f539 columnFamilyName c 2024-11-24T09:43:42,371 DEBUG [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:42,371 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(327): Store=14d2e44cf0df0f7e799ac7ef3309f539/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:42,371 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,372 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,373 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,374 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,374 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,375 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:42,376 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,377 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 14d2e44cf0df0f7e799ac7ef3309f539; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74465156, jitterRate=0.10961729288101196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:42,377 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:42,377 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 14d2e44cf0df0f7e799ac7ef3309f539: Running coprocessor pre-open hook at 1732441422364Writing region info on filesystem at 1732441422364Initializing all the Stores at 1732441422366 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441422366Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441422366Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441422366Cleaning up temporary data from old regions at 1732441422374 (+8 ms)Running coprocessor post-open hooks at 1732441422377 (+3 ms)Region opened successfully at 1732441422377 2024-11-24T09:43:42,392 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 14d2e44cf0df0f7e799ac7ef3309f539 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-24T09:43:42,393 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 14d2e44cf0df0f7e799ac7ef3309f539/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:42,708 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-24T09:43:42,708 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-24T09:43:42,710 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-24T09:43:42,710 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-24T09:43:42,711 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T09:43:42,711 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-24T09:43:42,712 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-24T09:43:42,712 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-24T09:43:42,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-24T09:43:42,714 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-24T09:43:42,862 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T09:43:43,394 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 14d2e44cf0df0f7e799ac7ef3309f539/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:43,565 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T09:43:44,395 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 14d2e44cf0df0f7e799ac7ef3309f539/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:45,396 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 14d2e44cf0df0f7e799ac7ef3309f539/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:46,397 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 14d2e44cf0df0f7e799ac7ef3309f539/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:47,398 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 14d2e44cf0df0f7e799ac7ef3309f539/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:48,398 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 14d2e44cf0df0f7e799ac7ef3309f539/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:49,399 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 14d2e44cf0df0f7e799ac7ef3309f539/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:50,400 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 14d2e44cf0df0f7e799ac7ef3309f539/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:51,401 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 14d2e44cf0df0f7e799ac7ef3309f539/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:51,403 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 14d2e44cf0df0f7e799ac7ef3309f539: 2024-11-24T09:43:51,403 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. 2024-11-24T09:43:51,418 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 14d2e44cf0df0f7e799ac7ef3309f539: 2024-11-24T09:43:51,418 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-24T09:43:51,418 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 14d2e44cf0df0f7e799ac7ef3309f539, disabling compactions & flushes 2024-11-24T09:43:51,418 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. 2024-11-24T09:43:51,418 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. 2024-11-24T09:43:51,418 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. after waiting 0 ms 2024-11-24T09:43:51,418 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. 2024-11-24T09:43:51,419 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. 2024-11-24T09:43:51,419 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. 2024-11-24T09:43:51,419 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 14d2e44cf0df0f7e799ac7ef3309f539: Waiting for close lock at 1732441431418Running coprocessor pre-close hooks at 1732441431418Disabling compacts and flushes for region at 1732441431418Disabling writes for close at 1732441431418Writing region close event to WAL at 1732441431419 (+1 ms)Running coprocessor post-close hooks at 1732441431419Closed at 1732441431419 2024-11-24T09:43:51,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741908_1086 (size=2685) 2024-11-24T09:43:51,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741908_1086 (size=2685) 2024-11-24T09:43:51,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741908_1086 (size=2685) 2024-11-24T09:43:51,436 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286, size=2.6 K (2685bytes) 2024-11-24T09:43:51,436 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286 2024-11-24T09:43:51,436 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286 after 0ms 2024-11-24T09:43:51,439 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:51,439 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286 took 3ms 2024-11-24T09:43:51,441 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286 so closing down 2024-11-24T09:43:51,441 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:51,442 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732441422286.temp 2024-11-24T09:43:51,443 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000004-wal.1732441422286.temp 2024-11-24T09:43:51,443 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:51,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741909_1087 (size=2094) 2024-11-24T09:43:51,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741909_1087 (size=2094) 2024-11-24T09:43:51,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741909_1087 (size=2094) 2024-11-24T09:43:51,450 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000004-wal.1732441422286.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-24T09:43:51,451 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000004-wal.1732441422286.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000026 2024-11-24T09:43:51,451 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286, size=2.6 K, length=2685, corrupted=false, cancelled=false 2024-11-24T09:43:51,451 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286, journal: Splitting hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286, size=2.6 K (2685bytes) at 1732441431436Finishing writing output for hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286 so closing down at 1732441431441 (+5 ms)Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000004-wal.1732441422286.temp at 1732441431443 (+2 ms)3 split writer threads finished at 1732441431444 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000004-wal.1732441422286.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732441431450 (+6 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000004-wal.1732441422286.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000026 at 1732441431451 (+1 ms)Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286, size=2.6 K, length=2685, corrupted=false, cancelled=false at 1732441431451 2024-11-24T09:43:51,453 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441422286 to hdfs://localhost:36549/hbase/oldWALs/wal.1732441422286 2024-11-24T09:43:51,454 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000026 2024-11-24T09:43:51,454 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:51,455 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:51,468 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441431455, exclude list is [], retry=0 2024-11-24T09:43:51,470 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:51,470 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:51,470 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:51,472 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441431455 2024-11-24T09:43:51,472 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:51,472 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 14d2e44cf0df0f7e799ac7ef3309f539, NAME => 'testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:51,473 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,473 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:51,473 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,473 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,475 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,475 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14d2e44cf0df0f7e799ac7ef3309f539 columnFamilyName a 2024-11-24T09:43:51,475 DEBUG [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:51,476 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(327): Store=14d2e44cf0df0f7e799ac7ef3309f539/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:51,476 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,477 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14d2e44cf0df0f7e799ac7ef3309f539 columnFamilyName b 2024-11-24T09:43:51,477 DEBUG [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:51,477 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(327): Store=14d2e44cf0df0f7e799ac7ef3309f539/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:51,477 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,478 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14d2e44cf0df0f7e799ac7ef3309f539 columnFamilyName c 2024-11-24T09:43:51,478 DEBUG [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:51,478 INFO [StoreOpener-14d2e44cf0df0f7e799ac7ef3309f539-1 {}] regionserver.HStore(327): Store=14d2e44cf0df0f7e799ac7ef3309f539/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:51,479 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,479 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,481 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,482 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000026 2024-11-24T09:43:51,484 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:51,486 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000026 2024-11-24T09:43:51,486 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 14d2e44cf0df0f7e799ac7ef3309f539 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-24T09:43:51,503 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/.tmp/a/0422a4e2d77b4c0bb951c82d93485dec is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732441431409/Put/seqid=0 2024-11-24T09:43:51,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741911_1089 (size=5523) 2024-11-24T09:43:51,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741911_1089 (size=5523) 2024-11-24T09:43:51,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741911_1089 (size=5523) 2024-11-24T09:43:51,510 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/.tmp/a/0422a4e2d77b4c0bb951c82d93485dec 2024-11-24T09:43:51,529 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/.tmp/b/271596cd6577426893cc174aad224288 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732441431404/Put/seqid=0 2024-11-24T09:43:51,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741912_1090 (size=5524) 2024-11-24T09:43:51,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741912_1090 (size=5524) 2024-11-24T09:43:51,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741912_1090 (size=5524) 2024-11-24T09:43:51,536 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/.tmp/b/271596cd6577426893cc174aad224288 2024-11-24T09:43:51,554 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/.tmp/c/4783689f3cc1430785857d9eda9d233b is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732441431406/Put/seqid=0 2024-11-24T09:43:51,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741913_1091 (size=5457) 2024-11-24T09:43:51,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741913_1091 (size=5457) 2024-11-24T09:43:51,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741913_1091 (size=5457) 2024-11-24T09:43:51,561 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/.tmp/c/4783689f3cc1430785857d9eda9d233b 2024-11-24T09:43:51,568 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/.tmp/a/0422a4e2d77b4c0bb951c82d93485dec as hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/a/0422a4e2d77b4c0bb951c82d93485dec 2024-11-24T09:43:51,568 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T09:43:51,574 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/a/0422a4e2d77b4c0bb951c82d93485dec, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T09:43:51,575 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/.tmp/b/271596cd6577426893cc174aad224288 as hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/b/271596cd6577426893cc174aad224288 2024-11-24T09:43:51,581 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/b/271596cd6577426893cc174aad224288, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T09:43:51,582 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/.tmp/c/4783689f3cc1430785857d9eda9d233b as hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/c/4783689f3cc1430785857d9eda9d233b 2024-11-24T09:43:51,588 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/c/4783689f3cc1430785857d9eda9d233b, entries=6, sequenceid=26, filesize=5.3 K 2024-11-24T09:43:51,588 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 14d2e44cf0df0f7e799ac7ef3309f539 in 102ms, sequenceid=26, compaction requested=false; wal=null 2024-11-24T09:43:51,589 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/0000000000000000026 2024-11-24T09:43:51,590 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,591 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,591 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:51,593 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,596 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsAfterAbortingFlush/14d2e44cf0df0f7e799ac7ef3309f539/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-24T09:43:51,598 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 14d2e44cf0df0f7e799ac7ef3309f539; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62540622, jitterRate=-0.068072110414505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:51,598 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 14d2e44cf0df0f7e799ac7ef3309f539 2024-11-24T09:43:51,598 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 14d2e44cf0df0f7e799ac7ef3309f539: Running coprocessor pre-open hook at 1732441431473Writing region info on filesystem at 1732441431473Initializing all the Stores at 1732441431474 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441431474Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441431474Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441431474Obtaining lock to block concurrent updates at 1732441431486 (+12 ms)Preparing flush snapshotting stores in 14d2e44cf0df0f7e799ac7ef3309f539 at 1732441431486Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732441431486Flushing stores of testReplayEditsAfterAbortingFlush,,1732441422220.14d2e44cf0df0f7e799ac7ef3309f539. at 1732441431486Flushing 14d2e44cf0df0f7e799ac7ef3309f539/a: creating writer at 1732441431486Flushing 14d2e44cf0df0f7e799ac7ef3309f539/a: appending metadata at 1732441431503 (+17 ms)Flushing 14d2e44cf0df0f7e799ac7ef3309f539/a: closing flushed file at 1732441431503Flushing 14d2e44cf0df0f7e799ac7ef3309f539/b: creating writer at 1732441431515 (+12 ms)Flushing 14d2e44cf0df0f7e799ac7ef3309f539/b: appending metadata at 1732441431529 (+14 ms)Flushing 14d2e44cf0df0f7e799ac7ef3309f539/b: closing flushed file at 1732441431529Flushing 14d2e44cf0df0f7e799ac7ef3309f539/c: creating writer at 1732441431541 (+12 ms)Flushing 14d2e44cf0df0f7e799ac7ef3309f539/c: appending metadata at 1732441431553 (+12 ms)Flushing 14d2e44cf0df0f7e799ac7ef3309f539/c: closing flushed file at 1732441431553Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14405dab: reopening flushed file at 1732441431567 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dcbccb7: reopening flushed file at 1732441431574 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27a4a3ff: reopening flushed file at 1732441431581 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 14d2e44cf0df0f7e799ac7ef3309f539 in 102ms, sequenceid=26, compaction requested=false; wal=null at 1732441431588 (+7 ms)Cleaning up temporary data from old regions at 1732441431591 (+3 ms)Running coprocessor post-open hooks at 1732441431598 (+7 ms)Region opened successfully at 1732441431598 2024-11-24T09:43:51,618 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=415 (was 415), OpenFileDescriptor=1184 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=263 (was 283), ProcessCount=11 (was 11), AvailableMemoryMB=7542 (was 7611) 2024-11-24T09:43:51,618 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-11-24T09:43:51,629 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1184, MaxFileDescriptor=1048576, SystemLoadAverage=263, ProcessCount=11, AvailableMemoryMB=7542 2024-11-24T09:43:51,629 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-11-24T09:43:51,642 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:51,644 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:51,644 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:51,647 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-40451179, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/hregion-40451179, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:51,658 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-40451179/hregion-40451179.1732441431647, exclude list is [], retry=0 2024-11-24T09:43:51,661 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:51,661 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:51,661 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:51,663 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-40451179/hregion-40451179.1732441431647 2024-11-24T09:43:51,664 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591)] 2024-11-24T09:43:51,664 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1b584a796325bc1dad3b53f64e99f2e1, NAME => 'testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36549/hbase 2024-11-24T09:43:51,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741915_1093 (size=61) 2024-11-24T09:43:51,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741915_1093 (size=61) 2024-11-24T09:43:51,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741915_1093 (size=61) 2024-11-24T09:43:51,674 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:51,675 INFO [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,676 INFO [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b584a796325bc1dad3b53f64e99f2e1 columnFamilyName a 2024-11-24T09:43:51,676 DEBUG [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:51,677 INFO [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] regionserver.HStore(327): Store=1b584a796325bc1dad3b53f64e99f2e1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:51,677 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,677 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,678 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,678 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,678 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,680 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,682 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:51,682 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1b584a796325bc1dad3b53f64e99f2e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61588051, jitterRate=-0.08226652443408966}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:43:51,683 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1b584a796325bc1dad3b53f64e99f2e1: Writing region info on filesystem at 1732441431674Initializing all the Stores at 1732441431674Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441431675 (+1 ms)Cleaning up temporary data from old regions at 1732441431678 (+3 ms)Region opened successfully at 1732441431683 (+5 ms) 2024-11-24T09:43:51,683 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1b584a796325bc1dad3b53f64e99f2e1, disabling compactions & flushes 2024-11-24T09:43:51,683 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1. 2024-11-24T09:43:51,683 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1. 2024-11-24T09:43:51,683 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1. after waiting 0 ms 2024-11-24T09:43:51,683 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1. 2024-11-24T09:43:51,684 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1. 2024-11-24T09:43:51,684 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1b584a796325bc1dad3b53f64e99f2e1: Waiting for close lock at 1732441431683Disabling compacts and flushes for region at 1732441431683Disabling writes for close at 1732441431683Writing region close event to WAL at 1732441431684 (+1 ms)Closed at 1732441431684 2024-11-24T09:43:51,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741914_1092 (size=95) 2024-11-24T09:43:51,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741914_1092 (size=95) 2024-11-24T09:43:51,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741914_1092 (size=95) 2024-11-24T09:43:51,689 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:51,690 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-40451179:(num 1732441431647) 2024-11-24T09:43:51,690 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:51,692 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:51,705 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692, exclude list is [], retry=0 2024-11-24T09:43:51,708 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:51,708 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:51,709 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:51,710 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692 2024-11-24T09:43:51,710 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:51,710 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b584a796325bc1dad3b53f64e99f2e1, NAME => 'testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:51,710 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:51,711 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,711 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,712 INFO [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,713 INFO [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b584a796325bc1dad3b53f64e99f2e1 columnFamilyName a 2024-11-24T09:43:51,713 DEBUG [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:51,714 INFO [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] regionserver.HStore(327): Store=1b584a796325bc1dad3b53f64e99f2e1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:51,714 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,715 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,716 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,716 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,716 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,718 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:51,718 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1b584a796325bc1dad3b53f64e99f2e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74504167, jitterRate=0.11019860208034515}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:43:51,719 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1b584a796325bc1dad3b53f64e99f2e1: Writing region info on filesystem at 1732441431711Initializing all the Stores at 1732441431712 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441431712Cleaning up temporary data from old regions at 1732441431716 (+4 ms)Region opened successfully at 1732441431719 (+3 ms) 2024-11-24T09:43:51,726 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1b584a796325bc1dad3b53f64e99f2e1, disabling compactions & flushes 2024-11-24T09:43:51,726 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1. 2024-11-24T09:43:51,727 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1. 2024-11-24T09:43:51,727 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1. after waiting 0 ms 2024-11-24T09:43:51,727 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1. 2024-11-24T09:43:51,727 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1. 2024-11-24T09:43:51,727 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1. 2024-11-24T09:43:51,727 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1b584a796325bc1dad3b53f64e99f2e1: Waiting for close lock at 1732441431726Disabling compacts and flushes for region at 1732441431726Disabling writes for close at 1732441431727 (+1 ms)Writing region close event to WAL at 1732441431727Closed at 1732441431727 2024-11-24T09:43:51,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741916_1094 (size=1050) 2024-11-24T09:43:51,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741916_1094 (size=1050) 2024-11-24T09:43:51,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741916_1094 (size=1050) 2024-11-24T09:43:51,753 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692, size=1.0 K (1050bytes) 2024-11-24T09:43:51,753 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692 2024-11-24T09:43:51,754 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692 after 1ms 2024-11-24T09:43:51,757 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:51,757 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692 took 4ms 2024-11-24T09:43:51,760 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692 so closing down 2024-11-24T09:43:51,760 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:51,761 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732441431692.temp 2024-11-24T09:43:51,763 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000003-wal.1732441431692.temp 2024-11-24T09:43:51,763 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:51,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741917_1095 (size=1050) 2024-11-24T09:43:51,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741917_1095 (size=1050) 2024-11-24T09:43:51,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741917_1095 (size=1050) 2024-11-24T09:43:51,778 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000003-wal.1732441431692.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-24T09:43:51,780 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000003-wal.1732441431692.temp to hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000012 2024-11-24T09:43:51,780 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-11-24T09:43:51,780 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692, journal: Splitting hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692, size=1.0 K (1050bytes) at 1732441431753Finishing writing output for hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692 so closing down at 1732441431760 (+7 ms)Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000003-wal.1732441431692.temp at 1732441431763 (+3 ms)3 split writer threads finished at 1732441431763Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000003-wal.1732441431692.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732441431778 (+15 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000003-wal.1732441431692.temp to hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000012 at 1732441431780 (+2 ms)Processed 10 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1732441431780 2024-11-24T09:43:51,782 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441431692 to hdfs://localhost:36549/hbase/oldWALs/wal.1732441431692 2024-11-24T09:43:51,783 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000012 2024-11-24T09:43:51,786 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:52,099 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:52,101 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:52,113 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441432101, exclude list is [], retry=0 2024-11-24T09:43:52,116 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:52,116 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:52,116 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:52,118 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441432101 2024-11-24T09:43:52,118 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591)] 2024-11-24T09:43:52,118 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b584a796325bc1dad3b53f64e99f2e1, NAME => 'testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:52,118 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:52,118 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,118 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,122 INFO [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,124 INFO [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b584a796325bc1dad3b53f64e99f2e1 columnFamilyName a 2024-11-24T09:43:52,124 DEBUG [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:52,125 INFO [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] regionserver.HStore(327): Store=1b584a796325bc1dad3b53f64e99f2e1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:52,125 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,126 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,127 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,128 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000012 2024-11-24T09:43:52,130 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:52,131 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000012 2024-11-24T09:43:52,131 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1b584a796325bc1dad3b53f64e99f2e1 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-24T09:43:52,147 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/.tmp/a/4fff9871908445388e80e4ccebfea96d is 79, key is testDatalossWhenInputError/a:x0/1732441431719/Put/seqid=0 2024-11-24T09:43:52,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741919_1097 (size=5808) 2024-11-24T09:43:52,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741919_1097 (size=5808) 2024-11-24T09:43:52,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741919_1097 (size=5808) 2024-11-24T09:43:52,154 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/.tmp/a/4fff9871908445388e80e4ccebfea96d 2024-11-24T09:43:52,164 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/.tmp/a/4fff9871908445388e80e4ccebfea96d as hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/a/4fff9871908445388e80e4ccebfea96d 2024-11-24T09:43:52,174 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/a/4fff9871908445388e80e4ccebfea96d, entries=10, sequenceid=12, filesize=5.7 K 2024-11-24T09:43:52,174 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 1b584a796325bc1dad3b53f64e99f2e1 in 43ms, sequenceid=12, compaction requested=false; wal=null 2024-11-24T09:43:52,175 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/0000000000000000012 2024-11-24T09:43:52,176 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,176 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,179 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,181 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-24T09:43:52,182 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1b584a796325bc1dad3b53f64e99f2e1; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73311154, jitterRate=0.09242132306098938}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:43:52,183 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1b584a796325bc1dad3b53f64e99f2e1: Writing region info on filesystem at 1732441432119Initializing all the Stores at 1732441432122 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441432122Obtaining lock to block concurrent updates at 1732441432131 (+9 ms)Preparing flush snapshotting stores in 1b584a796325bc1dad3b53f64e99f2e1 at 1732441432131Finished memstore snapshotting testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732441432131Flushing stores of testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1. at 1732441432131Flushing 1b584a796325bc1dad3b53f64e99f2e1/a: creating writer at 1732441432131Flushing 1b584a796325bc1dad3b53f64e99f2e1/a: appending metadata at 1732441432147 (+16 ms)Flushing 1b584a796325bc1dad3b53f64e99f2e1/a: closing flushed file at 1732441432147Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@359c44a0: reopening flushed file at 1732441432163 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 1b584a796325bc1dad3b53f64e99f2e1 in 43ms, sequenceid=12, compaction requested=false; wal=null at 1732441432174 (+11 ms)Cleaning up temporary data from old regions at 1732441432176 (+2 ms)Region opened successfully at 1732441432183 (+7 ms) 2024-11-24T09:43:52,186 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b584a796325bc1dad3b53f64e99f2e1, NAME => 'testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:52,186 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732441431643.1b584a796325bc1dad3b53f64e99f2e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:52,186 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,186 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,187 INFO [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,188 INFO [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b584a796325bc1dad3b53f64e99f2e1 columnFamilyName a 2024-11-24T09:43:52,188 DEBUG [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:52,193 DEBUG [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/a/4fff9871908445388e80e4ccebfea96d 2024-11-24T09:43:52,193 INFO [StoreOpener-1b584a796325bc1dad3b53f64e99f2e1-1 {}] regionserver.HStore(327): Store=1b584a796325bc1dad3b53f64e99f2e1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:52,193 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,194 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,196 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,196 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,196 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,198 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1b584a796325bc1dad3b53f64e99f2e1 2024-11-24T09:43:52,201 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testDatalossWhenInputError/1b584a796325bc1dad3b53f64e99f2e1/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-24T09:43:52,202 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1b584a796325bc1dad3b53f64e99f2e1; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69678140, jitterRate=0.03828519582748413}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:43:52,202 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1b584a796325bc1dad3b53f64e99f2e1: Writing region info on filesystem at 1732441432186Initializing all the Stores at 1732441432187 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441432187Cleaning up temporary data from old regions at 1732441432196 (+9 ms)Region opened successfully at 1732441432202 (+6 ms) 2024-11-24T09:43:52,221 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=424 (was 415) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:39352 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:40122 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:52812 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:47740 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-880808591-172.17.0.2-1732441398809:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:47502 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:33584 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1184) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=263 (was 263), ProcessCount=11 (was 11), AvailableMemoryMB=7532 (was 7542) 2024-11-24T09:43:52,221 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T09:43:52,233 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=424, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=263, ProcessCount=11, AvailableMemoryMB=7532 2024-11-24T09:43:52,233 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T09:43:52,246 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:52,248 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:52,249 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:52,251 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-94491864, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/hregion-94491864, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:52,263 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-94491864/hregion-94491864.1732441432251, exclude list is [], retry=0 2024-11-24T09:43:52,265 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:52,266 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:52,266 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:52,268 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-94491864/hregion-94491864.1732441432251 2024-11-24T09:43:52,269 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:43:52,269 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ea128656e4a08082560ace15a320afba, NAME => 'testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36549/hbase 2024-11-24T09:43:52,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741921_1099 (size=63) 2024-11-24T09:43:52,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741921_1099 (size=63) 2024-11-24T09:43:52,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741921_1099 (size=63) 2024-11-24T09:43:52,281 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:52,283 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,284 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea128656e4a08082560ace15a320afba columnFamilyName a 2024-11-24T09:43:52,284 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:52,285 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(327): Store=ea128656e4a08082560ace15a320afba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:52,285 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,286 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea128656e4a08082560ace15a320afba columnFamilyName b 2024-11-24T09:43:52,286 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:52,286 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(327): Store=ea128656e4a08082560ace15a320afba/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:52,287 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,288 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea128656e4a08082560ace15a320afba columnFamilyName c 2024-11-24T09:43:52,288 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:52,288 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(327): Store=ea128656e4a08082560ace15a320afba/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:52,289 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,289 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,290 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,291 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,291 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,291 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:52,293 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,295 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:52,296 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea128656e4a08082560ace15a320afba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69701835, jitterRate=0.03863827884197235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:52,296 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea128656e4a08082560ace15a320afba: Writing region info on filesystem at 1732441432281Initializing all the Stores at 1732441432282 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441432282Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441432282Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441432283 (+1 ms)Cleaning up temporary data from old regions at 1732441432291 (+8 ms)Region opened successfully at 1732441432296 (+5 ms) 2024-11-24T09:43:52,297 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea128656e4a08082560ace15a320afba, disabling compactions & flushes 2024-11-24T09:43:52,297 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba. 2024-11-24T09:43:52,297 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba. 2024-11-24T09:43:52,297 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba. after waiting 0 ms 2024-11-24T09:43:52,297 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba. 2024-11-24T09:43:52,297 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba. 2024-11-24T09:43:52,297 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea128656e4a08082560ace15a320afba: Waiting for close lock at 1732441432297Disabling compacts and flushes for region at 1732441432297Disabling writes for close at 1732441432297Writing region close event to WAL at 1732441432297Closed at 1732441432297 2024-11-24T09:43:52,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741920_1098 (size=95) 2024-11-24T09:43:52,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741920_1098 (size=95) 2024-11-24T09:43:52,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741920_1098 (size=95) 2024-11-24T09:43:52,306 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:52,306 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-94491864:(num 1732441432251) 2024-11-24T09:43:52,306 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:52,307 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:52,322 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308, exclude list is [], retry=0 2024-11-24T09:43:52,325 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:52,325 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:52,325 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:52,335 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308 2024-11-24T09:43:52,335 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591)] 2024-11-24T09:43:52,335 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ea128656e4a08082560ace15a320afba, NAME => 'testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:52,336 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:52,336 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,336 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,337 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,338 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea128656e4a08082560ace15a320afba columnFamilyName a 2024-11-24T09:43:52,338 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:52,339 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(327): Store=ea128656e4a08082560ace15a320afba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:52,339 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,339 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea128656e4a08082560ace15a320afba columnFamilyName b 2024-11-24T09:43:52,339 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:52,340 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(327): Store=ea128656e4a08082560ace15a320afba/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:52,340 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,341 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea128656e4a08082560ace15a320afba columnFamilyName c 2024-11-24T09:43:52,341 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:52,341 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(327): Store=ea128656e4a08082560ace15a320afba/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:52,342 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,342 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,343 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,344 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,344 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,345 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:52,346 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:52,347 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea128656e4a08082560ace15a320afba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65083242, jitterRate=-0.030184119939804077}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:52,348 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea128656e4a08082560ace15a320afba: Writing region info on filesystem at 1732441432336Initializing all the Stores at 1732441432337 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441432337Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441432337Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441432337Cleaning up temporary data from old regions at 1732441432344 (+7 ms)Region opened successfully at 1732441432348 (+4 ms) 2024-11-24T09:43:52,353 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732441432353/Put/seqid=0 2024-11-24T09:43:52,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741923_1101 (size=4875) 2024-11-24T09:43:52,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741923_1101 (size=4875) 2024-11-24T09:43:52,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741923_1101 (size=4875) 2024-11-24T09:43:52,363 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732441432363/Put/seqid=0 2024-11-24T09:43:52,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741924_1102 (size=4875) 2024-11-24T09:43:52,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741924_1102 (size=4875) 2024-11-24T09:43:52,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741924_1102 (size=4875) 2024-11-24T09:43:52,378 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732441432378/Put/seqid=0 2024-11-24T09:43:52,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741925_1103 (size=4875) 2024-11-24T09:43:52,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741925_1103 (size=4875) 2024-11-24T09:43:52,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741925_1103 (size=4875) 2024-11-24T09:43:52,385 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in ea128656e4a08082560ace15a320afba/a 2024-11-24T09:43:52,389 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-24T09:43:52,389 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T09:43:52,389 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in ea128656e4a08082560ace15a320afba/a 2024-11-24T09:43:52,394 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-24T09:43:52,394 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T09:43:52,394 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in ea128656e4a08082560ace15a320afba/a 2024-11-24T09:43:52,397 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-24T09:43:52,397 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T09:43:52,398 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea128656e4a08082560ace15a320afba 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-24T09:43:52,413 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/.tmp/a/73bf1c63f15c433f85b5c8374e09b0f5 is 55, key is testCompactedBulkLoadedFiles/a:a/1732441432348/Put/seqid=0 2024-11-24T09:43:52,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741926_1104 (size=5107) 2024-11-24T09:43:52,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741926_1104 (size=5107) 2024-11-24T09:43:52,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741926_1104 (size=5107) 2024-11-24T09:43:52,420 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/.tmp/a/73bf1c63f15c433f85b5c8374e09b0f5 2024-11-24T09:43:52,425 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/.tmp/a/73bf1c63f15c433f85b5c8374e09b0f5 as hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/73bf1c63f15c433f85b5c8374e09b0f5 2024-11-24T09:43:52,430 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/73bf1c63f15c433f85b5c8374e09b0f5, entries=1, sequenceid=4, filesize=5.0 K 2024-11-24T09:43:52,431 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for ea128656e4a08082560ace15a320afba in 34ms, sequenceid=4, compaction requested=false 2024-11-24T09:43:52,431 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ea128656e4a08082560ace15a320afba: 2024-11-24T09:43:52,432 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/85b6635143ca44509de69bf525309895_SeqId_4_ 2024-11-24T09:43:52,433 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_ 2024-11-24T09:43:52,434 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/72350287f96c4c71923afbd5dc901500_SeqId_4_ 2024-11-24T09:43:52,434 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile0 into ea128656e4a08082560ace15a320afba/a as hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/85b6635143ca44509de69bf525309895_SeqId_4_ - updating store file list. 2024-11-24T09:43:52,438 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 85b6635143ca44509de69bf525309895_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T09:43:52,439 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/85b6635143ca44509de69bf525309895_SeqId_4_ into ea128656e4a08082560ace15a320afba/a 2024-11-24T09:43:52,439 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile0 into ea128656e4a08082560ace15a320afba/a (new location: hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/85b6635143ca44509de69bf525309895_SeqId_4_) 2024-11-24T09:43:52,440 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile1 into ea128656e4a08082560ace15a320afba/a as hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_ - updating store file list. 2024-11-24T09:43:52,443 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T09:43:52,444 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_ into ea128656e4a08082560ace15a320afba/a 2024-11-24T09:43:52,444 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile1 into ea128656e4a08082560ace15a320afba/a (new location: hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_) 2024-11-24T09:43:52,445 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile2 into ea128656e4a08082560ace15a320afba/a as hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/72350287f96c4c71923afbd5dc901500_SeqId_4_ - updating store file list. 2024-11-24T09:43:52,449 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 72350287f96c4c71923afbd5dc901500_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T09:43:52,449 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/72350287f96c4c71923afbd5dc901500_SeqId_4_ into ea128656e4a08082560ace15a320afba/a 2024-11-24T09:43:52,449 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36549/hbase/testCompactedBulkLoadedFiles/hfile2 into ea128656e4a08082560ace15a320afba/a (new location: hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/72350287f96c4c71923afbd5dc901500_SeqId_4_) 2024-11-24T09:43:52,455 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T09:43:52,455 DEBUG [Time-limited test {}] regionserver.HStore(1541): ea128656e4a08082560ace15a320afba/a is initiating major compaction (all files) 2024-11-24T09:43:52,455 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ea128656e4a08082560ace15a320afba/a in testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba. 2024-11-24T09:43:52,456 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/73bf1c63f15c433f85b5c8374e09b0f5, hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/85b6635143ca44509de69bf525309895_SeqId_4_, hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_, hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/72350287f96c4c71923afbd5dc901500_SeqId_4_] into tmpdir=hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/.tmp, totalSize=19.3 K 2024-11-24T09:43:52,456 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 73bf1c63f15c433f85b5c8374e09b0f5, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732441432348 2024-11-24T09:43:52,457 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 85b6635143ca44509de69bf525309895_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T09:43:52,457 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T09:43:52,457 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 72350287f96c4c71923afbd5dc901500_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T09:43:52,468 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/.tmp/a/ecec49eeacf14c968f70e14d33166a6b is 55, key is testCompactedBulkLoadedFiles/a:a/1732441432348/Put/seqid=0 2024-11-24T09:43:52,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741927_1105 (size=6154) 2024-11-24T09:43:52,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741927_1105 (size=6154) 2024-11-24T09:43:52,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741927_1105 (size=6154) 2024-11-24T09:43:52,483 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/.tmp/a/ecec49eeacf14c968f70e14d33166a6b as hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/ecec49eeacf14c968f70e14d33166a6b 2024-11-24T09:43:52,490 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in ea128656e4a08082560ace15a320afba/a of ea128656e4a08082560ace15a320afba into ecec49eeacf14c968f70e14d33166a6b(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T09:43:52,490 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ea128656e4a08082560ace15a320afba: 2024-11-24T09:43:52,490 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T09:43:52,491 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T09:43:52,523 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308, size=0 (0bytes) 2024-11-24T09:43:52,523 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308 might be still open, length is 0 2024-11-24T09:43:52,523 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308 2024-11-24T09:43:52,523 WARN [IPC Server handler 0 on default port 36549 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-24T09:43:52,523 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308 after 0ms 2024-11-24T09:43:52,708 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-24T09:43:52,708 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-24T09:43:52,710 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-24T09:43:52,710 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-24T09:43:54,526 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:39386 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37953:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39386 dst: /127.0.0.1:37953 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37953 remote=/127.0.0.1:39386]. Total timeout mills is 60000, 57966 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:54,527 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:47788 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:33653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47788 dst: /127.0.0.1:33653 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:54,527 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:47520 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37499:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47520 dst: /127.0.0.1:37499 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:54,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741922_1106 (size=1174) 2024-11-24T09:43:54,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741922_1106 (size=1174) 2024-11-24T09:43:54,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741922_1106 (size=1174) 2024-11-24T09:43:56,525 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308 after 4001ms 2024-11-24T09:43:56,530 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:56,530 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308 took 4008ms 2024-11-24T09:43:56,533 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308; continuing. 2024-11-24T09:43:56,533 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308 so closing down 2024-11-24T09:43:56,533 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:56,536 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732441432308.temp 2024-11-24T09:43:56,538 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000003-wal.1732441432308.temp 2024-11-24T09:43:56,538 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:56,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741928_1107 (size=549) 2024-11-24T09:43:56,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741928_1107 (size=549) 2024-11-24T09:43:56,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741928_1107 (size=549) 2024-11-24T09:43:56,547 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000003-wal.1732441432308.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T09:43:56,548 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000003-wal.1732441432308.temp to hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000008 2024-11-24T09:43:56,549 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T09:43:56,549 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308, journal: Splitting hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308, size=0 (0bytes) at 1732441432523Finishing writing output for hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308 so closing down at 1732441436533 (+4010 ms)Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000003-wal.1732441432308.temp at 1732441436538 (+5 ms)3 split writer threads finished at 1732441436538Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000003-wal.1732441432308.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732441436547 (+9 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000003-wal.1732441432308.temp to hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000008 at 1732441436548 (+1 ms)Processed 5 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308, size=0, length=0, corrupted=false, cancelled=false at 1732441436549 (+1 ms) 2024-11-24T09:43:56,551 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308 to hdfs://localhost:36549/hbase/oldWALs/wal.1732441432308 2024-11-24T09:43:56,552 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000008 2024-11-24T09:43:56,553 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:56,555 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:56,566 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441436555, exclude list is [], retry=0 2024-11-24T09:43:56,569 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:56,569 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:56,569 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:56,570 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441436555 2024-11-24T09:43:56,571 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:35185:35185)] 2024-11-24T09:43:56,571 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ea128656e4a08082560ace15a320afba, NAME => 'testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:56,571 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:56,571 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:56,571 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:56,572 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea128656e4a08082560ace15a320afba 2024-11-24T09:43:56,573 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea128656e4a08082560ace15a320afba columnFamilyName a 2024-11-24T09:43:56,573 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:56,579 DEBUG [StoreFileOpener-ea128656e4a08082560ace15a320afba-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T09:43:56,579 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_ 2024-11-24T09:43:56,583 DEBUG [StoreFileOpener-ea128656e4a08082560ace15a320afba-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 72350287f96c4c71923afbd5dc901500_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T09:43:56,583 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/72350287f96c4c71923afbd5dc901500_SeqId_4_ 2024-11-24T09:43:56,587 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/73bf1c63f15c433f85b5c8374e09b0f5 2024-11-24T09:43:56,590 DEBUG [StoreFileOpener-ea128656e4a08082560ace15a320afba-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 85b6635143ca44509de69bf525309895_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T09:43:56,590 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/85b6635143ca44509de69bf525309895_SeqId_4_ 2024-11-24T09:43:56,594 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/ecec49eeacf14c968f70e14d33166a6b 2024-11-24T09:43:56,594 WARN [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1f71eaa5 2024-11-24T09:43:56,594 WARN [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/72350287f96c4c71923afbd5dc901500_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1f71eaa5 2024-11-24T09:43:56,594 WARN [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/73bf1c63f15c433f85b5c8374e09b0f5 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1f71eaa5 2024-11-24T09:43:56,594 WARN [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/85b6635143ca44509de69bf525309895_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1f71eaa5 2024-11-24T09:43:56,594 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_, hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/72350287f96c4c71923afbd5dc901500_SeqId_4_, hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/73bf1c63f15c433f85b5c8374e09b0f5, hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/85b6635143ca44509de69bf525309895_SeqId_4_] to archive 2024-11-24T09:43:56,595 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T09:43:56,597 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_ to hdfs://localhost:36549/hbase/archive/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_ 2024-11-24T09:43:56,598 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/72350287f96c4c71923afbd5dc901500_SeqId_4_ to hdfs://localhost:36549/hbase/archive/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/72350287f96c4c71923afbd5dc901500_SeqId_4_ 2024-11-24T09:43:56,599 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/73bf1c63f15c433f85b5c8374e09b0f5 to hdfs://localhost:36549/hbase/archive/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/73bf1c63f15c433f85b5c8374e09b0f5 2024-11-24T09:43:56,600 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/85b6635143ca44509de69bf525309895_SeqId_4_ to hdfs://localhost:36549/hbase/archive/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/a/85b6635143ca44509de69bf525309895_SeqId_4_ 2024-11-24T09:43:56,600 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(327): Store=ea128656e4a08082560ace15a320afba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:56,600 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea128656e4a08082560ace15a320afba 2024-11-24T09:43:56,601 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea128656e4a08082560ace15a320afba columnFamilyName b 2024-11-24T09:43:56,601 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:56,601 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(327): Store=ea128656e4a08082560ace15a320afba/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:56,601 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea128656e4a08082560ace15a320afba 2024-11-24T09:43:56,602 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea128656e4a08082560ace15a320afba columnFamilyName c 2024-11-24T09:43:56,602 DEBUG [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:56,602 INFO [StoreOpener-ea128656e4a08082560ace15a320afba-1 {}] regionserver.HStore(327): Store=ea128656e4a08082560ace15a320afba/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:56,602 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:56,603 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba 2024-11-24T09:43:56,604 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba 2024-11-24T09:43:56,605 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000008 2024-11-24T09:43:56,606 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:56,608 DEBUG [Time-limited test {}] regionserver.HRegion(5836): ea128656e4a08082560ace15a320afba : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "ea128656e4a08082560ace15a320afba" family_name: "a" compaction_input: "73bf1c63f15c433f85b5c8374e09b0f5" compaction_input: "85b6635143ca44509de69bf525309895_SeqId_4_" compaction_input: "24af8d331abe4224b2ee9a35cf19b7a0_SeqId_4_" compaction_input: "72350287f96c4c71923afbd5dc901500_SeqId_4_" compaction_output: "ecec49eeacf14c968f70e14d33166a6b" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-24T09:43:56,608 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-24T09:43:56,608 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000008 2024-11-24T09:43:56,609 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/0000000000000000008 2024-11-24T09:43:56,609 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:56,609 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:56,610 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:56,611 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea128656e4a08082560ace15a320afba 2024-11-24T09:43:56,613 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testCompactedBulkLoadedFiles/ea128656e4a08082560ace15a320afba/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T09:43:56,614 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea128656e4a08082560ace15a320afba; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68881315, jitterRate=0.026411578059196472}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:56,615 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea128656e4a08082560ace15a320afba: Writing region info on filesystem at 1732441436571Initializing all the Stores at 1732441436572 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441436572Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441436572Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441436572Cleaning up temporary data from old regions at 1732441436609 (+37 ms)Region opened successfully at 1732441436614 (+5 ms) 2024-11-24T09:43:56,617 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea128656e4a08082560ace15a320afba, disabling compactions & flushes 2024-11-24T09:43:56,617 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba. 2024-11-24T09:43:56,617 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba. 2024-11-24T09:43:56,617 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba. after waiting 0 ms 2024-11-24T09:43:56,617 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba. 2024-11-24T09:43:56,618 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732441432247.ea128656e4a08082560ace15a320afba. 2024-11-24T09:43:56,618 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea128656e4a08082560ace15a320afba: Waiting for close lock at 1732441436617Disabling compacts and flushes for region at 1732441436617Disabling writes for close at 1732441436617Writing region close event to WAL at 1732441436618 (+1 ms)Closed at 1732441436618 2024-11-24T09:43:56,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741929_1108 (size=95) 2024-11-24T09:43:56,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741929_1108 (size=95) 2024-11-24T09:43:56,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741929_1108 (size=95) 2024-11-24T09:43:56,622 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:56,622 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732441436555) 2024-11-24T09:43:56,635 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=438 (was 424) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1915071097_22 at /127.0.0.1:47582 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1915071097_22 at /127.0.0.1:39472 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:36549 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:36549 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1915071097_22 at /127.0.0.1:47878 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=242 (was 263), ProcessCount=11 (was 11), AvailableMemoryMB=7519 (was 7532) 2024-11-24T09:43:56,635 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-24T09:43:56,646 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=242, ProcessCount=11, AvailableMemoryMB=7518 2024-11-24T09:43:56,646 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-24T09:43:56,660 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:56,661 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:43:56,662 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:43:56,664 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-26841625, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/hregion-26841625, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:56,676 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-26841625/hregion-26841625.1732441436664, exclude list is [], retry=0 2024-11-24T09:43:56,678 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:56,679 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:56,680 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:56,681 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-26841625/hregion-26841625.1732441436664 2024-11-24T09:43:56,681 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:35185:35185)] 2024-11-24T09:43:56,682 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dbc2705f410308fb8b515c78f5ff7beb, NAME => 'testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36549/hbase 2024-11-24T09:43:56,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741931_1110 (size=67) 2024-11-24T09:43:56,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741931_1110 (size=67) 2024-11-24T09:43:56,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741931_1110 (size=67) 2024-11-24T09:43:56,693 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:56,695 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,696 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc2705f410308fb8b515c78f5ff7beb columnFamilyName a 2024-11-24T09:43:56,696 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:56,696 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(327): Store=dbc2705f410308fb8b515c78f5ff7beb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:56,696 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,697 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc2705f410308fb8b515c78f5ff7beb columnFamilyName b 2024-11-24T09:43:56,697 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:56,698 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(327): Store=dbc2705f410308fb8b515c78f5ff7beb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:56,698 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,699 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc2705f410308fb8b515c78f5ff7beb columnFamilyName c 2024-11-24T09:43:56,699 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:56,699 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(327): Store=dbc2705f410308fb8b515c78f5ff7beb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:56,700 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,700 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,700 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,701 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,701 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,702 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:56,703 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,705 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:43:56,705 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dbc2705f410308fb8b515c78f5ff7beb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59768057, jitterRate=-0.10938654839992523}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:56,705 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dbc2705f410308fb8b515c78f5ff7beb: Writing region info on filesystem at 1732441436694Initializing all the Stores at 1732441436694Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441436694Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441436694Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441436694Cleaning up temporary data from old regions at 1732441436701 (+7 ms)Region opened successfully at 1732441436705 (+4 ms) 2024-11-24T09:43:56,705 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dbc2705f410308fb8b515c78f5ff7beb, disabling compactions & flushes 2024-11-24T09:43:56,705 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:43:56,705 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:43:56,705 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. after waiting 0 ms 2024-11-24T09:43:56,706 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:43:56,706 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:43:56,706 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dbc2705f410308fb8b515c78f5ff7beb: Waiting for close lock at 1732441436705Disabling compacts and flushes for region at 1732441436705Disabling writes for close at 1732441436706 (+1 ms)Writing region close event to WAL at 1732441436706Closed at 1732441436706 2024-11-24T09:43:56,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741930_1109 (size=95) 2024-11-24T09:43:56,708 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-26841625/hregion-26841625.1732441436664 not finished, retry = 0 2024-11-24T09:43:56,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741930_1109 (size=95) 2024-11-24T09:43:56,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741930_1109 (size=95) 2024-11-24T09:43:56,811 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:43:56,811 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-26841625:(num 1732441436664) 2024-11-24T09:43:56,811 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:56,813 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:56,826 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813, exclude list is [], retry=0 2024-11-24T09:43:56,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:56,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:56,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:56,831 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813 2024-11-24T09:43:56,831 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591)] 2024-11-24T09:43:56,832 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dbc2705f410308fb8b515c78f5ff7beb, NAME => 'testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:56,832 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:56,832 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,832 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,833 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,834 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc2705f410308fb8b515c78f5ff7beb columnFamilyName a 2024-11-24T09:43:56,834 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:56,835 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(327): Store=dbc2705f410308fb8b515c78f5ff7beb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:56,835 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,836 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc2705f410308fb8b515c78f5ff7beb columnFamilyName b 2024-11-24T09:43:56,836 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:56,836 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(327): Store=dbc2705f410308fb8b515c78f5ff7beb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:56,836 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,837 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc2705f410308fb8b515c78f5ff7beb columnFamilyName c 2024-11-24T09:43:56,837 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:56,837 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(327): Store=dbc2705f410308fb8b515c78f5ff7beb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:56,837 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,838 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,839 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,840 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,840 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,841 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:56,842 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,843 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dbc2705f410308fb8b515c78f5ff7beb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66178245, jitterRate=-0.013867303729057312}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:56,843 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dbc2705f410308fb8b515c78f5ff7beb: Writing region info on filesystem at 1732441436832Initializing all the Stores at 1732441436833 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441436833Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441436833Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441436833Cleaning up temporary data from old regions at 1732441436840 (+7 ms)Region opened successfully at 1732441436843 (+3 ms) 2024-11-24T09:43:56,851 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dbc2705f410308fb8b515c78f5ff7beb 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T09:43:56,866 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/a/ff672afa1f8148ceafe207ffc3894d32 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732441436843/Put/seqid=0 2024-11-24T09:43:56,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741933_1112 (size=5958) 2024-11-24T09:43:56,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741933_1112 (size=5958) 2024-11-24T09:43:56,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741933_1112 (size=5958) 2024-11-24T09:43:56,875 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/a/ff672afa1f8148ceafe207ffc3894d32 2024-11-24T09:43:56,880 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/a/ff672afa1f8148ceafe207ffc3894d32 as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/a/ff672afa1f8148ceafe207ffc3894d32 2024-11-24T09:43:56,885 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/a/ff672afa1f8148ceafe207ffc3894d32, entries=10, sequenceid=13, filesize=5.8 K 2024-11-24T09:43:56,886 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for dbc2705f410308fb8b515c78f5ff7beb in 35ms, sequenceid=13, compaction requested=false 2024-11-24T09:43:56,886 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for dbc2705f410308fb8b515c78f5ff7beb: 2024-11-24T09:43:56,907 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dbc2705f410308fb8b515c78f5ff7beb, disabling compactions & flushes 2024-11-24T09:43:56,907 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:43:56,907 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:43:56,907 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. after waiting 0 ms 2024-11-24T09:43:56,907 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:43:56,908 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:43:56,908 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:43:56,908 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dbc2705f410308fb8b515c78f5ff7beb: Waiting for close lock at 1732441436907Disabling compacts and flushes for region at 1732441436907Disabling writes for close at 1732441436907Writing region close event to WAL at 1732441436908 (+1 ms)Closed at 1732441436908 2024-11-24T09:43:56,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741932_1111 (size=3344) 2024-11-24T09:43:56,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741932_1111 (size=3344) 2024-11-24T09:43:56,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741932_1111 (size=3344) 2024-11-24T09:43:56,926 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813, size=3.3 K (3344bytes) 2024-11-24T09:43:56,926 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813 2024-11-24T09:43:56,926 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813 after 0ms 2024-11-24T09:43:56,928 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:56,928 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813 took 2ms 2024-11-24T09:43:56,930 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813 so closing down 2024-11-24T09:43:56,930 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:43:56,931 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732441436813.temp 2024-11-24T09:43:56,932 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000003-wal.1732441436813.temp 2024-11-24T09:43:56,933 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:43:56,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741934_1113 (size=2944) 2024-11-24T09:43:56,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741934_1113 (size=2944) 2024-11-24T09:43:56,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741934_1113 (size=2944) 2024-11-24T09:43:56,939 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000003-wal.1732441436813.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T09:43:56,941 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000003-wal.1732441436813.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000035 2024-11-24T09:43:56,941 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813, size=3.3 K, length=3344, corrupted=false, cancelled=false 2024-11-24T09:43:56,941 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813, journal: Splitting hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813, size=3.3 K (3344bytes) at 1732441436926Finishing writing output for hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813 so closing down at 1732441436930 (+4 ms)Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000003-wal.1732441436813.temp at 1732441436932 (+2 ms)3 split writer threads finished at 1732441436933 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000003-wal.1732441436813.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732441436939 (+6 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000003-wal.1732441436813.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000035 at 1732441436941 (+2 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813, size=3.3 K, length=3344, corrupted=false, cancelled=false at 1732441436941 2024-11-24T09:43:56,942 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436813 to hdfs://localhost:36549/hbase/oldWALs/wal.1732441436813 2024-11-24T09:43:56,943 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000035 2024-11-24T09:43:56,944 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:43:56,945 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:43:56,952 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T09:43:56,965 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946, exclude list is [], retry=0 2024-11-24T09:43:56,967 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:43:56,968 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:43:56,968 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:43:56,970 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946 2024-11-24T09:43:56,970 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:42101:42101),(127.0.0.1/127.0.0.1:40591:40591)] 2024-11-24T09:43:56,970 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dbc2705f410308fb8b515c78f5ff7beb, NAME => 'testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:43:56,970 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:43:56,970 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,970 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,975 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,977 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc2705f410308fb8b515c78f5ff7beb columnFamilyName a 2024-11-24T09:43:56,977 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:56,983 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/a/ff672afa1f8148ceafe207ffc3894d32 2024-11-24T09:43:56,983 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(327): Store=dbc2705f410308fb8b515c78f5ff7beb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:56,983 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,984 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc2705f410308fb8b515c78f5ff7beb columnFamilyName b 2024-11-24T09:43:56,984 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:56,985 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(327): Store=dbc2705f410308fb8b515c78f5ff7beb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:56,985 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,986 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc2705f410308fb8b515c78f5ff7beb columnFamilyName c 2024-11-24T09:43:56,986 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:43:56,986 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(327): Store=dbc2705f410308fb8b515c78f5ff7beb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:43:56,986 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,987 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,988 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:56,989 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000035 2024-11-24T09:43:56,991 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:43:56,992 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000035 2024-11-24T09:43:56,992 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dbc2705f410308fb8b515c78f5ff7beb 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-24T09:43:57,007 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/b/cf44dc0492ce414f830f6f0f85630d14 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732441436886/Put/seqid=0 2024-11-24T09:43:57,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741936_1115 (size=5958) 2024-11-24T09:43:57,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741936_1115 (size=5958) 2024-11-24T09:43:57,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741936_1115 (size=5958) 2024-11-24T09:43:57,014 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/b/cf44dc0492ce414f830f6f0f85630d14 2024-11-24T09:43:57,036 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/c/51f2bb8757f54544a76a494cd013f57d is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732441436894/Put/seqid=0 2024-11-24T09:43:57,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741937_1116 (size=5958) 2024-11-24T09:43:57,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741937_1116 (size=5958) 2024-11-24T09:43:57,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741937_1116 (size=5958) 2024-11-24T09:43:57,043 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/c/51f2bb8757f54544a76a494cd013f57d 2024-11-24T09:43:57,048 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/b/cf44dc0492ce414f830f6f0f85630d14 as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/b/cf44dc0492ce414f830f6f0f85630d14 2024-11-24T09:43:57,053 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/b/cf44dc0492ce414f830f6f0f85630d14, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T09:43:57,054 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/c/51f2bb8757f54544a76a494cd013f57d as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/c/51f2bb8757f54544a76a494cd013f57d 2024-11-24T09:43:57,059 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/c/51f2bb8757f54544a76a494cd013f57d, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T09:43:57,059 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for dbc2705f410308fb8b515c78f5ff7beb in 67ms, sequenceid=35, compaction requested=false; wal=null 2024-11-24T09:43:57,060 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000035 2024-11-24T09:43:57,061 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:57,061 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:57,062 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:43:57,063 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:43:57,065 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-24T09:43:57,066 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dbc2705f410308fb8b515c78f5ff7beb; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63981097, jitterRate=-0.0466073602437973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:43:57,066 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dbc2705f410308fb8b515c78f5ff7beb: Writing region info on filesystem at 1732441436971Initializing all the Stores at 1732441436972 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441436972Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441436975 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441436975Obtaining lock to block concurrent updates at 1732441436992 (+17 ms)Preparing flush snapshotting stores in dbc2705f410308fb8b515c78f5ff7beb at 1732441436992Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732441436993 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. at 1732441436993Flushing dbc2705f410308fb8b515c78f5ff7beb/b: creating writer at 1732441436993Flushing dbc2705f410308fb8b515c78f5ff7beb/b: appending metadata at 1732441437006 (+13 ms)Flushing dbc2705f410308fb8b515c78f5ff7beb/b: closing flushed file at 1732441437006Flushing dbc2705f410308fb8b515c78f5ff7beb/c: creating writer at 1732441437019 (+13 ms)Flushing dbc2705f410308fb8b515c78f5ff7beb/c: appending metadata at 1732441437036 (+17 ms)Flushing dbc2705f410308fb8b515c78f5ff7beb/c: closing flushed file at 1732441437036Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3903cb00: reopening flushed file at 1732441437047 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e175857: reopening flushed file at 1732441437053 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for dbc2705f410308fb8b515c78f5ff7beb in 67ms, sequenceid=35, compaction requested=false; wal=null at 1732441437059 (+6 ms)Cleaning up temporary data from old regions at 1732441437061 (+2 ms)Region opened successfully at 1732441437066 (+5 ms) 2024-11-24T09:43:57,128 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946, size=0 (0bytes) 2024-11-24T09:43:57,128 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946 might be still open, length is 0 2024-11-24T09:43:57,128 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946 2024-11-24T09:43:57,128 WARN [IPC Server handler 1 on default port 36549 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-24T09:43:57,129 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946 after 1ms 2024-11-24T09:43:57,523 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:39542 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37953:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39542 dst: /127.0.0.1:37953 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37953 remote=/127.0.0.1:39542]. Total timeout mills is 60000, 59568 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:57,524 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:47610 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37499:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47610 dst: /127.0.0.1:37499 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:57,524 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_845225009_22 at /127.0.0.1:47926 [Receiving block BP-880808591-172.17.0.2-1732441398809:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:33653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47926 dst: /127.0.0.1:33653 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:43:57,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741935_1117 (size=2936) 2024-11-24T09:43:57,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741935_1117 (size=2936) 2024-11-24T09:44:01,130 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946 after 4002ms 2024-11-24T09:44:01,195 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:44:01,195 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946 took 4067ms 2024-11-24T09:44:01,200 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946; continuing. 2024-11-24T09:44:01,200 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946 so closing down 2024-11-24T09:44:01,201 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:44:01,203 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732441436946.temp 2024-11-24T09:44:01,204 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000037-wal.1732441436946.temp 2024-11-24T09:44:01,205 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:44:01,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741938_1118 (size=2944) 2024-11-24T09:44:01,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741938_1118 (size=2944) 2024-11-24T09:44:01,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741938_1118 (size=2944) 2024-11-24T09:44:01,213 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000037-wal.1732441436946.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T09:44:01,215 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000037-wal.1732441436946.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000066 2024-11-24T09:44:01,215 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T09:44:01,215 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946, journal: Splitting hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946, size=0 (0bytes) at 1732441437128Finishing writing output for hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946 so closing down at 1732441441200 (+4072 ms)Creating recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000037-wal.1732441436946.temp at 1732441441204 (+4 ms)3 split writer threads finished at 1732441441205 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000037-wal.1732441436946.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732441441214 (+9 ms)Rename recovered edits hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000037-wal.1732441436946.temp to hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000066 at 1732441441215 (+1 ms)Processed 30 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946, size=0, length=0, corrupted=false, cancelled=false at 1732441441215 2024-11-24T09:44:01,217 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946 to hdfs://localhost:36549/hbase/oldWALs/wal.1732441436946 2024-11-24T09:44:01,218 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000066 2024-11-24T09:44:01,218 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:44:01,220 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36549/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659, archiveDir=hdfs://localhost:36549/hbase/oldWALs, maxLogs=32 2024-11-24T09:44:01,233 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441441220, exclude list is [], retry=0 2024-11-24T09:44:01,235 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37953,DS-3b590ec6-b75f-4c23-ad18-76aa7cd9c977,DISK] 2024-11-24T09:44:01,236 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33653,DS-cece125c-6b20-4102-926b-c1801fb0cd76,DISK] 2024-11-24T09:44:01,236 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37499,DS-a72cf2c2-0ba4-476a-b150-f0fdf03fabd4,DISK] 2024-11-24T09:44:01,237 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441441220 2024-11-24T09:44:01,237 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:40591:40591),(127.0.0.1/127.0.0.1:42101:42101)] 2024-11-24T09:44:01,238 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:44:01,241 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:44:01,242 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc2705f410308fb8b515c78f5ff7beb columnFamilyName a 2024-11-24T09:44:01,242 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:44:01,247 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/a/ff672afa1f8148ceafe207ffc3894d32 2024-11-24T09:44:01,247 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(327): Store=dbc2705f410308fb8b515c78f5ff7beb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:44:01,248 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:44:01,248 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc2705f410308fb8b515c78f5ff7beb columnFamilyName b 2024-11-24T09:44:01,248 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:44:01,254 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/b/cf44dc0492ce414f830f6f0f85630d14 2024-11-24T09:44:01,254 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(327): Store=dbc2705f410308fb8b515c78f5ff7beb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:44:01,255 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:44:01,255 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc2705f410308fb8b515c78f5ff7beb columnFamilyName c 2024-11-24T09:44:01,255 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:44:01,261 DEBUG [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/c/51f2bb8757f54544a76a494cd013f57d 2024-11-24T09:44:01,262 INFO [StoreOpener-dbc2705f410308fb8b515c78f5ff7beb-1 {}] regionserver.HStore(327): Store=dbc2705f410308fb8b515c78f5ff7beb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:44:01,262 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:44:01,263 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:44:01,264 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:44:01,265 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000066 2024-11-24T09:44:01,267 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T09:44:01,271 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000066 2024-11-24T09:44:01,272 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dbc2705f410308fb8b515c78f5ff7beb 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T09:44:01,286 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/a/1fdcca23d54e47fb86847a2056465f9d is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732441437072/Put/seqid=0 2024-11-24T09:44:01,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741940_1120 (size=5958) 2024-11-24T09:44:01,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741940_1120 (size=5958) 2024-11-24T09:44:01,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741940_1120 (size=5958) 2024-11-24T09:44:01,295 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/a/1fdcca23d54e47fb86847a2056465f9d 2024-11-24T09:44:01,316 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/b/c46e8fda6f2847c28eccb27283bbe1bf is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732441437079/Put/seqid=0 2024-11-24T09:44:01,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741941_1121 (size=5958) 2024-11-24T09:44:01,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741941_1121 (size=5958) 2024-11-24T09:44:01,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741941_1121 (size=5958) 2024-11-24T09:44:01,326 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/b/c46e8fda6f2847c28eccb27283bbe1bf 2024-11-24T09:44:01,344 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/c/d8a9094be8eb42f2af394e0092fb683e is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732441437085/Put/seqid=0 2024-11-24T09:44:01,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741942_1122 (size=5958) 2024-11-24T09:44:01,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741942_1122 (size=5958) 2024-11-24T09:44:01,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741942_1122 (size=5958) 2024-11-24T09:44:01,351 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/c/d8a9094be8eb42f2af394e0092fb683e 2024-11-24T09:44:01,357 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/a/1fdcca23d54e47fb86847a2056465f9d as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/a/1fdcca23d54e47fb86847a2056465f9d 2024-11-24T09:44:01,362 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/a/1fdcca23d54e47fb86847a2056465f9d, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T09:44:01,363 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/b/c46e8fda6f2847c28eccb27283bbe1bf as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/b/c46e8fda6f2847c28eccb27283bbe1bf 2024-11-24T09:44:01,368 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/b/c46e8fda6f2847c28eccb27283bbe1bf, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T09:44:01,369 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/.tmp/c/d8a9094be8eb42f2af394e0092fb683e as hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/c/d8a9094be8eb42f2af394e0092fb683e 2024-11-24T09:44:01,373 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/c/d8a9094be8eb42f2af394e0092fb683e, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T09:44:01,373 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for dbc2705f410308fb8b515c78f5ff7beb in 102ms, sequenceid=66, compaction requested=false; wal=null 2024-11-24T09:44:01,374 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/0000000000000000066 2024-11-24T09:44:01,375 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:44:01,375 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:44:01,376 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:44:01,377 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dbc2705f410308fb8b515c78f5ff7beb 2024-11-24T09:44:01,378 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/hbase/data/default/testReplayEditsWrittenViaHRegion/dbc2705f410308fb8b515c78f5ff7beb/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-24T09:44:01,379 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dbc2705f410308fb8b515c78f5ff7beb; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74860093, jitterRate=0.11550231277942657}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:44:01,380 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dbc2705f410308fb8b515c78f5ff7beb: Writing region info on filesystem at 1732441441238Initializing all the Stores at 1732441441239 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441441239Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441441240 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441441240Obtaining lock to block concurrent updates at 1732441441272 (+32 ms)Preparing flush snapshotting stores in dbc2705f410308fb8b515c78f5ff7beb at 1732441441272Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732441441272Flushing stores of testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. at 1732441441272Flushing dbc2705f410308fb8b515c78f5ff7beb/a: creating writer at 1732441441272Flushing dbc2705f410308fb8b515c78f5ff7beb/a: appending metadata at 1732441441286 (+14 ms)Flushing dbc2705f410308fb8b515c78f5ff7beb/a: closing flushed file at 1732441441286Flushing dbc2705f410308fb8b515c78f5ff7beb/b: creating writer at 1732441441300 (+14 ms)Flushing dbc2705f410308fb8b515c78f5ff7beb/b: appending metadata at 1732441441316 (+16 ms)Flushing dbc2705f410308fb8b515c78f5ff7beb/b: closing flushed file at 1732441441316Flushing dbc2705f410308fb8b515c78f5ff7beb/c: creating writer at 1732441441330 (+14 ms)Flushing dbc2705f410308fb8b515c78f5ff7beb/c: appending metadata at 1732441441344 (+14 ms)Flushing dbc2705f410308fb8b515c78f5ff7beb/c: closing flushed file at 1732441441344Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54a6b5a4: reopening flushed file at 1732441441356 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1374ff0: reopening flushed file at 1732441441362 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b244d23: reopening flushed file at 1732441441368 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for dbc2705f410308fb8b515c78f5ff7beb in 102ms, sequenceid=66, compaction requested=false; wal=null at 1732441441373 (+5 ms)Cleaning up temporary data from old regions at 1732441441375 (+2 ms)Region opened successfully at 1732441441379 (+4 ms) 2024-11-24T09:44:01,392 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dbc2705f410308fb8b515c78f5ff7beb, disabling compactions & flushes 2024-11-24T09:44:01,392 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:44:01,392 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:44:01,392 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. after waiting 0 ms 2024-11-24T09:44:01,392 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:44:01,393 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732441436660.dbc2705f410308fb8b515c78f5ff7beb. 2024-11-24T09:44:01,393 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dbc2705f410308fb8b515c78f5ff7beb: Waiting for close lock at 1732441441392Disabling compacts and flushes for region at 1732441441392Disabling writes for close at 1732441441392Writing region close event to WAL at 1732441441393 (+1 ms)Closed at 1732441441393 2024-11-24T09:44:01,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741939_1119 (size=95) 2024-11-24T09:44:01,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741939_1119 (size=95) 2024-11-24T09:44:01,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741939_1119 (size=95) 2024-11-24T09:44:01,398 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:44:01,398 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732441441220) 2024-11-24T09:44:01,411 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=445 (was 438) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-175351302_22 at /127.0.0.1:47664 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:36549 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-175351302_22 at /127.0.0.1:39576 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-175351302_22 at /127.0.0.1:47958 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:36549 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=263 (was 242) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7487 (was 7518) 2024-11-24T09:44:01,411 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-24T09:44:01,411 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T09:44:01,411 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T09:44:01,411 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:44:01,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:44:01,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:44:01,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T09:44:01,412 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T09:44:01,412 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=387325396, stopped=false 2024-11-24T09:44:01,412 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ba35ada4783f,39325,1732441402377 2024-11-24T09:44:01,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T09:44:01,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T09:44:01,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T09:44:01,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:44:01,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:44:01,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:44:01,447 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T09:44:01,468 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T09:44:01,468 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:44:01,468 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:44:01,468 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:44:01,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:44:01,468 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba35ada4783f,42075,1732441403106' ***** 2024-11-24T09:44:01,468 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T09:44:01,468 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba35ada4783f,34867,1732441403238' ***** 2024-11-24T09:44:01,468 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T09:44:01,468 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:44:01,468 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T09:44:01,468 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T09:44:01,468 INFO [RS:1;ba35ada4783f:34867 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T09:44:01,468 INFO [RS:0;ba35ada4783f:42075 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T09:44:01,468 INFO [RS:0;ba35ada4783f:42075 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T09:44:01,468 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T09:44:01,469 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T09:44:01,469 INFO [RS:1;ba35ada4783f:34867 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T09:44:01,469 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(959): stopping server ba35ada4783f,42075,1732441403106 2024-11-24T09:44:01,469 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(3091): Received CLOSE for f1ee6750d931ca2321055907c01bc718 2024-11-24T09:44:01,469 INFO [RS:0;ba35ada4783f:42075 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T09:44:01,469 INFO [RS:0;ba35ada4783f:42075 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ba35ada4783f:42075. 2024-11-24T09:44:01,469 DEBUG [RS:0;ba35ada4783f:42075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:44:01,469 DEBUG [RS:0;ba35ada4783f:42075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:44:01,469 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T09:44:01,469 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T09:44:01,469 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T09:44:01,469 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T09:44:01,469 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(959): stopping server ba35ada4783f,34867,1732441403238 2024-11-24T09:44:01,470 INFO [RS:1;ba35ada4783f:34867 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T09:44:01,470 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T09:44:01,470 INFO [RS:1;ba35ada4783f:34867 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ba35ada4783f:34867. 2024-11-24T09:44:01,470 DEBUG [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T09:44:01,470 DEBUG [RS:1;ba35ada4783f:34867 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:44:01,470 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f1ee6750d931ca2321055907c01bc718, disabling compactions & flushes 2024-11-24T09:44:01,470 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T09:44:01,470 DEBUG [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T09:44:01,470 DEBUG [RS:1;ba35ada4783f:34867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:44:01,470 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:44:01,470 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T09:44:01,470 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:44:01,470 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T09:44:01,470 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T09:44:01,470 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. after waiting 0 ms 2024-11-24T09:44:01,470 DEBUG [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(1325): Online Regions={f1ee6750d931ca2321055907c01bc718=testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718.} 2024-11-24T09:44:01,470 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T09:44:01,470 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:44:01,470 DEBUG [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(1351): Waiting on f1ee6750d931ca2321055907c01bc718 2024-11-24T09:44:01,470 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T09:44:01,470 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-24T09:44:01,476 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/.tmp/info/9c067cc2d83e4d91bdd2a572eac30445 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718./info:regioninfo/1732441421727/Put/seqid=0 2024-11-24T09:44:01,478 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f1ee6750d931ca2321055907c01bc718/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-24T09:44:01,479 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:44:01,479 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f1ee6750d931ca2321055907c01bc718: Waiting for close lock at 1732441441470Running coprocessor pre-close hooks at 1732441441470Disabling compacts and flushes for region at 1732441441470Disabling writes for close at 1732441441470Writing region close event to WAL at 1732441441475 (+5 ms)Running coprocessor post-close hooks at 1732441441479 (+4 ms)Closed at 1732441441479 2024-11-24T09:44:01,479 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441417620.f1ee6750d931ca2321055907c01bc718. 2024-11-24T09:44:01,482 INFO [regionserver/ba35ada4783f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:44:01,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741943_1123 (size=6778) 2024-11-24T09:44:01,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741943_1123 (size=6778) 2024-11-24T09:44:01,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741943_1123 (size=6778) 2024-11-24T09:44:01,483 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/.tmp/info/9c067cc2d83e4d91bdd2a572eac30445 2024-11-24T09:44:01,483 INFO [regionserver/ba35ada4783f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:44:01,489 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/.tmp/info/9c067cc2d83e4d91bdd2a572eac30445 as hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/info/9c067cc2d83e4d91bdd2a572eac30445 2024-11-24T09:44:01,494 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/info/9c067cc2d83e4d91bdd2a572eac30445, entries=8, sequenceid=23, filesize=6.6 K 2024-11-24T09:44:01,494 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false 2024-11-24T09:44:01,498 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-24T09:44:01,499 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T09:44:01,499 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T09:44:01,499 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732441441470Running coprocessor pre-close hooks at 1732441441470Disabling compacts and flushes for region at 1732441441470Disabling writes for close at 1732441441470Obtaining lock to block concurrent updates at 1732441441470Preparing flush snapshotting stores in 1588230740 at 1732441441470Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732441441470Flushing stores of hbase:meta,,1.1588230740 at 1732441441471 (+1 ms)Flushing 1588230740/info: creating writer at 1732441441471Flushing 1588230740/info: appending metadata at 1732441441476 (+5 ms)Flushing 1588230740/info: closing flushed file at 1732441441476Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65259e5: reopening flushed file at 1732441441488 (+12 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false at 1732441441494 (+6 ms)Writing region close event to WAL at 1732441441495 (+1 ms)Running coprocessor post-close hooks at 1732441441499 (+4 ms)Closed at 1732441441499 2024-11-24T09:44:01,499 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T09:44:01,670 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(976): stopping server ba35ada4783f,42075,1732441403106; all regions closed. 2024-11-24T09:44:01,670 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(976): stopping server ba35ada4783f,34867,1732441403238; all regions closed. 2024-11-24T09:44:01,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741893_1071 (size=1673) 2024-11-24T09:44:01,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741834_1010 (size=719) 2024-11-24T09:44:01,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741834_1010 (size=719) 2024-11-24T09:44:01,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741893_1071 (size=1673) 2024-11-24T09:44:01,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741893_1071 (size=1673) 2024-11-24T09:44:01,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741834_1010 (size=719) 2024-11-24T09:44:01,680 INFO [regionserver/ba35ada4783f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T09:44:01,680 INFO [regionserver/ba35ada4783f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T09:44:01,680 DEBUG [RS:1;ba35ada4783f:34867 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/oldWALs 2024-11-24T09:44:01,680 INFO [RS:1;ba35ada4783f:34867 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ba35ada4783f%2C34867%2C1732441403238:(num 1732441404802) 2024-11-24T09:44:01,680 DEBUG [RS:0;ba35ada4783f:42075 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/oldWALs 2024-11-24T09:44:01,680 DEBUG [RS:1;ba35ada4783f:34867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:44:01,680 INFO [RS:0;ba35ada4783f:42075 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ba35ada4783f%2C42075%2C1732441403106.meta:.meta(num 1732441420860) 2024-11-24T09:44:01,680 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:44:01,681 INFO [RS:1;ba35ada4783f:34867 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T09:44:01,681 INFO [RS:1;ba35ada4783f:34867 {}] hbase.ChoreService(370): Chore service for: regionserver/ba35ada4783f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T09:44:01,681 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T09:44:01,681 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T09:44:01,681 INFO [regionserver/ba35ada4783f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T09:44:01,681 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T09:44:01,681 INFO [RS:1;ba35ada4783f:34867 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T09:44:01,681 INFO [RS:1;ba35ada4783f:34867 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34867 2024-11-24T09:44:01,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741833_1009 (size=1712) 2024-11-24T09:44:01,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741833_1009 (size=1712) 2024-11-24T09:44:01,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741833_1009 (size=1712) 2024-11-24T09:44:01,685 DEBUG [RS:0;ba35ada4783f:42075 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/oldWALs 2024-11-24T09:44:01,685 INFO [RS:0;ba35ada4783f:42075 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ba35ada4783f%2C42075%2C1732441403106:(num 1732441404802) 2024-11-24T09:44:01,685 DEBUG [RS:0;ba35ada4783f:42075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:44:01,685 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:44:01,685 INFO [RS:0;ba35ada4783f:42075 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T09:44:01,685 INFO [RS:0;ba35ada4783f:42075 {}] hbase.ChoreService(370): Chore service for: regionserver/ba35ada4783f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T09:44:01,686 INFO [RS:0;ba35ada4783f:42075 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T09:44:01,686 INFO [regionserver/ba35ada4783f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T09:44:01,686 INFO [RS:0;ba35ada4783f:42075 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42075 2024-11-24T09:44:01,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T09:44:01,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba35ada4783f,34867,1732441403238 2024-11-24T09:44:01,714 INFO [RS:1;ba35ada4783f:34867 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T09:44:01,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba35ada4783f,42075,1732441403106 2024-11-24T09:44:01,722 INFO [RS:0;ba35ada4783f:42075 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T09:44:01,730 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba35ada4783f,34867,1732441403238] 2024-11-24T09:44:01,747 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba35ada4783f,34867,1732441403238 already deleted, retry=false 2024-11-24T09:44:01,747 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba35ada4783f,34867,1732441403238 expired; onlineServers=1 2024-11-24T09:44:01,747 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba35ada4783f,42075,1732441403106] 2024-11-24T09:44:01,755 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba35ada4783f,42075,1732441403106 already deleted, retry=false 2024-11-24T09:44:01,755 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba35ada4783f,42075,1732441403106 expired; onlineServers=0 2024-11-24T09:44:01,756 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ba35ada4783f,39325,1732441402377' ***** 2024-11-24T09:44:01,756 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T09:44:01,756 INFO [M:0;ba35ada4783f:39325 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T09:44:01,756 INFO [M:0;ba35ada4783f:39325 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T09:44:01,756 DEBUG [M:0;ba35ada4783f:39325 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T09:44:01,757 DEBUG [M:0;ba35ada4783f:39325 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T09:44:01,757 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T09:44:01,757 DEBUG [master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.large.0-1732441404425 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.large.0-1732441404425,5,FailOnTimeoutGroup] 2024-11-24T09:44:01,757 DEBUG [master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.small.0-1732441404426 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.small.0-1732441404426,5,FailOnTimeoutGroup] 2024-11-24T09:44:01,757 INFO [M:0;ba35ada4783f:39325 {}] hbase.ChoreService(370): Chore service for: master/ba35ada4783f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T09:44:01,757 INFO [M:0;ba35ada4783f:39325 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T09:44:01,758 DEBUG [M:0;ba35ada4783f:39325 {}] master.HMaster(1795): Stopping service threads 2024-11-24T09:44:01,758 INFO [M:0;ba35ada4783f:39325 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T09:44:01,758 INFO [M:0;ba35ada4783f:39325 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T09:44:01,759 INFO [M:0;ba35ada4783f:39325 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T09:44:01,759 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T09:44:01,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T09:44:01,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:44:01,764 DEBUG [M:0;ba35ada4783f:39325 {}] zookeeper.ZKUtil(347): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T09:44:01,765 WARN [M:0;ba35ada4783f:39325 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T09:44:01,766 INFO [M:0;ba35ada4783f:39325 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/.lastflushedseqids 2024-11-24T09:44:01,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741944_1124 (size=119) 2024-11-24T09:44:01,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741944_1124 (size=119) 2024-11-24T09:44:01,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741944_1124 (size=119) 2024-11-24T09:44:01,783 INFO [M:0;ba35ada4783f:39325 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T09:44:01,784 INFO [M:0;ba35ada4783f:39325 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T09:44:01,784 DEBUG [M:0;ba35ada4783f:39325 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T09:44:01,784 INFO [M:0;ba35ada4783f:39325 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:44:01,784 DEBUG [M:0;ba35ada4783f:39325 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:44:01,784 DEBUG [M:0;ba35ada4783f:39325 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T09:44:01,784 DEBUG [M:0;ba35ada4783f:39325 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:44:01,784 INFO [M:0;ba35ada4783f:39325 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-24T09:44:01,802 DEBUG [M:0;ba35ada4783f:39325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7e4ff998d08b42d0a3a781e7b686a0f1 is 82, key is hbase:meta,,1/info:regioninfo/1732441421021/Put/seqid=0 2024-11-24T09:44:01,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741945_1125 (size=6063) 2024-11-24T09:44:01,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741945_1125 (size=6063) 2024-11-24T09:44:01,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741945_1125 (size=6063) 2024-11-24T09:44:01,808 INFO [M:0;ba35ada4783f:39325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7e4ff998d08b42d0a3a781e7b686a0f1 2024-11-24T09:44:01,826 DEBUG [M:0;ba35ada4783f:39325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/22480453033649dda2cf35af8450b43b is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732441418060/Put/seqid=0 2024-11-24T09:44:01,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:44:01,830 INFO [RS:1;ba35ada4783f:34867 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T09:44:01,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34867-0x1016d837bd10002, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:44:01,831 INFO [RS:1;ba35ada4783f:34867 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba35ada4783f,34867,1732441403238; zookeeper connection closed. 2024-11-24T09:44:01,831 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3034627a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3034627a 2024-11-24T09:44:01,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741946_1126 (size=7907) 2024-11-24T09:44:01,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741946_1126 (size=7907) 2024-11-24T09:44:01,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741946_1126 (size=7907) 2024-11-24T09:44:01,832 INFO [M:0;ba35ada4783f:39325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/22480453033649dda2cf35af8450b43b 2024-11-24T09:44:01,836 INFO [M:0;ba35ada4783f:39325 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 22480453033649dda2cf35af8450b43b 2024-11-24T09:44:01,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:44:01,839 INFO [RS:0;ba35ada4783f:42075 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T09:44:01,839 INFO [RS:0;ba35ada4783f:42075 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba35ada4783f,42075,1732441403106; zookeeper connection closed. 2024-11-24T09:44:01,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x1016d837bd10001, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:44:01,839 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@270adad6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@270adad6 2024-11-24T09:44:01,840 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T09:44:01,850 DEBUG [M:0;ba35ada4783f:39325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e7b90cf671174103a11e368e75d108df is 69, key is ba35ada4783f,34867,1732441403238/rs:state/1732441404518/Put/seqid=0 2024-11-24T09:44:01,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741947_1127 (size=5440) 2024-11-24T09:44:01,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741947_1127 (size=5440) 2024-11-24T09:44:01,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741947_1127 (size=5440) 2024-11-24T09:44:01,856 INFO [M:0;ba35ada4783f:39325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e7b90cf671174103a11e368e75d108df 2024-11-24T09:44:01,860 INFO [M:0;ba35ada4783f:39325 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e7b90cf671174103a11e368e75d108df 2024-11-24T09:44:01,861 DEBUG [M:0;ba35ada4783f:39325 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7e4ff998d08b42d0a3a781e7b686a0f1 as hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7e4ff998d08b42d0a3a781e7b686a0f1 2024-11-24T09:44:01,865 INFO [M:0;ba35ada4783f:39325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7e4ff998d08b42d0a3a781e7b686a0f1, entries=14, sequenceid=207, filesize=5.9 K 2024-11-24T09:44:01,866 DEBUG [M:0;ba35ada4783f:39325 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/22480453033649dda2cf35af8450b43b as hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/22480453033649dda2cf35af8450b43b 2024-11-24T09:44:01,870 INFO [M:0;ba35ada4783f:39325 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 22480453033649dda2cf35af8450b43b 2024-11-24T09:44:01,870 INFO [M:0;ba35ada4783f:39325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/22480453033649dda2cf35af8450b43b, entries=21, sequenceid=207, filesize=7.7 K 2024-11-24T09:44:01,871 DEBUG [M:0;ba35ada4783f:39325 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e7b90cf671174103a11e368e75d108df as hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e7b90cf671174103a11e368e75d108df 2024-11-24T09:44:01,876 INFO [M:0;ba35ada4783f:39325 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e7b90cf671174103a11e368e75d108df 2024-11-24T09:44:01,876 INFO [M:0;ba35ada4783f:39325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36549/user/jenkins/test-data/d4d82c13-bf59-904a-5057-515624818f88/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e7b90cf671174103a11e368e75d108df, entries=3, sequenceid=207, filesize=5.3 K 2024-11-24T09:44:01,877 INFO [M:0;ba35ada4783f:39325 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=207, compaction requested=false 2024-11-24T09:44:01,879 INFO [M:0;ba35ada4783f:39325 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:44:01,879 DEBUG [M:0;ba35ada4783f:39325 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732441441784Disabling compacts and flushes for region at 1732441441784Disabling writes for close at 1732441441784Obtaining lock to block concurrent updates at 1732441441784Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732441441784Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1732441441785 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732441441785Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732441441785Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732441441801 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732441441802 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732441441812 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732441441826 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732441441826Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732441441836 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732441441850 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732441441850Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a9749a3: reopening flushed file at 1732441441860 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@767ea25b: reopening flushed file at 1732441441865 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e32176c: reopening flushed file at 1732441441870 (+5 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=207, compaction requested=false at 1732441441877 (+7 ms)Writing region close event to WAL at 1732441441878 (+1 ms)Closed at 1732441441878 2024-11-24T09:44:01,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37499 is added to blk_1073741830_1006 (size=69434) 2024-11-24T09:44:01,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37953 is added to blk_1073741830_1006 (size=69434) 2024-11-24T09:44:01,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33653 is added to blk_1073741830_1006 (size=69434) 2024-11-24T09:44:01,881 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T09:44:01,881 INFO [M:0;ba35ada4783f:39325 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T09:44:01,881 INFO [M:0;ba35ada4783f:39325 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39325 2024-11-24T09:44:01,882 INFO [M:0;ba35ada4783f:39325 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T09:44:02,006 INFO [M:0;ba35ada4783f:39325 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T09:44:02,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:44:02,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39325-0x1016d837bd10000, quorum=127.0.0.1:57552, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:44:02,013 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441431455 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441431455 (inode 16688) Holder DFSClient_NONMAPREDUCE_845225009_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441422219/wal.1732441431455 (inode 16688) Holder DFSClient_NONMAPREDUCE_845225009_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T09:44:02,017 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441422082 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441422082 (inode 16665) Holder DFSClient_NONMAPREDUCE_845225009_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441421792/wal.1732441422082 (inode 16665) Holder DFSClient_NONMAPREDUCE_845225009_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T09:44:02,018 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441413199/wal.1732441413294 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:44:02,021 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441432101 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441432101 (inode 16714) Holder DFSClient_NONMAPREDUCE_845225009_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441431642/wal.1732441432101 (inode 16714) Holder DFSClient_NONMAPREDUCE_845225009_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T09:44:02,022 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441432246/wal.1732441432308 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:44:02,022 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441436659/wal.1732441436946 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:44:02,022 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441406466/wal.1732441406538 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:44:02,025 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal.1732441406337 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal.1732441406337 (inode 16485) Holder DFSClient_NONMAPREDUCE_845225009_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441406164/wal.1732441406337 (inode 16485) Holder DFSClient_NONMAPREDUCE_845225009_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T09:44:02,029 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal.1732441405888 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal.1732441405888 (inode 16462) Holder DFSClient_NONMAPREDUCE_845225009_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441405633/wal.1732441405888 (inode 16462) Holder DFSClient_NONMAPREDUCE_845225009_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T09:44:02,036 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T09:44:02,039 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T09:44:02,039 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T09:44:02,039 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T09:44:02,039 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/hadoop.log.dir/,STOPPED} 2024-11-24T09:44:02,042 WARN [BP-880808591-172.17.0.2-1732441398809 heartbeating to localhost/127.0.0.1:36549 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T09:44:02,042 WARN [BP-880808591-172.17.0.2-1732441398809 heartbeating to localhost/127.0.0.1:36549 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-880808591-172.17.0.2-1732441398809 (Datanode Uuid 27220eb0-c1fd-4d9d-ab92-71a0aab887ed) service to localhost/127.0.0.1:36549 2024-11-24T09:44:02,042 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T09:44:02,042 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T09:44:02,044 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data5/current/BP-880808591-172.17.0.2-1732441398809 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T09:44:02,044 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data6/current/BP-880808591-172.17.0.2-1732441398809 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T09:44:02,045 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T09:44:02,051 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T09:44:02,052 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T09:44:02,052 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T09:44:02,052 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T09:44:02,052 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/hadoop.log.dir/,STOPPED} 2024-11-24T09:44:02,053 WARN [BP-880808591-172.17.0.2-1732441398809 heartbeating to localhost/127.0.0.1:36549 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T09:44:02,053 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T09:44:02,053 WARN [BP-880808591-172.17.0.2-1732441398809 heartbeating to localhost/127.0.0.1:36549 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-880808591-172.17.0.2-1732441398809 (Datanode Uuid 9024e8f8-076f-4b78-a55d-0d0a87f756bf) service to localhost/127.0.0.1:36549 2024-11-24T09:44:02,053 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T09:44:02,054 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data3/current/BP-880808591-172.17.0.2-1732441398809 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T09:44:02,055 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data4/current/BP-880808591-172.17.0.2-1732441398809 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T09:44:02,055 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T09:44:02,063 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T09:44:02,064 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T09:44:02,064 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T09:44:02,064 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T09:44:02,064 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/hadoop.log.dir/,STOPPED} 2024-11-24T09:44:02,065 WARN [BP-880808591-172.17.0.2-1732441398809 heartbeating to localhost/127.0.0.1:36549 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T09:44:02,065 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T09:44:02,065 WARN [BP-880808591-172.17.0.2-1732441398809 heartbeating to localhost/127.0.0.1:36549 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-880808591-172.17.0.2-1732441398809 (Datanode Uuid 9fa7c2d1-dbb5-474d-bb12-c3da4aaa534a) service to localhost/127.0.0.1:36549 2024-11-24T09:44:02,065 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T09:44:02,066 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data1/current/BP-880808591-172.17.0.2-1732441398809 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T09:44:02,066 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/cluster_1e3d208e-49da-9cf4-851c-2228dac46779/data/data2/current/BP-880808591-172.17.0.2-1732441398809 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T09:44:02,066 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T09:44:02,074 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T09:44:02,075 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T09:44:02,075 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T09:44:02,075 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T09:44:02,075 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa8c5b9-df39-f6d4-a15e-f65efa4e77fb/hadoop.log.dir/,STOPPED} 2024-11-24T09:44:02,088 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T09:44:02,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down