2024-11-16 02:34:15,824 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-16 02:34:15,835 main DEBUG Took 0.009297 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 02:34:15,836 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 02:34:15,836 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 02:34:15,837 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 02:34:15,838 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,845 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 02:34:15,858 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,859 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,860 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,860 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,861 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,861 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,862 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,862 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,863 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,863 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,864 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,864 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,865 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,865 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,866 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,866 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,866 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,867 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,867 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,868 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,868 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,869 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,869 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,869 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 02:34:15,870 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,870 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 02:34:15,872 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 02:34:15,873 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 02:34:15,875 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 02:34:15,875 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 02:34:15,876 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 02:34:15,877 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 02:34:15,885 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 02:34:15,888 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 02:34:15,890 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 02:34:15,890 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 02:34:15,891 main DEBUG createAppenders(={Console}) 2024-11-16 02:34:15,892 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-16 02:34:15,892 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-16 02:34:15,893 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-16 02:34:15,893 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 02:34:15,894 main DEBUG OutputStream closed 2024-11-16 02:34:15,894 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 02:34:15,894 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 02:34:15,894 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-16 02:34:15,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 02:34:15,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 02:34:15,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 02:34:15,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 02:34:15,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 02:34:15,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 02:34:15,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 02:34:15,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 02:34:15,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 02:34:15,977 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 02:34:15,977 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 02:34:15,977 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 02:34:15,977 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 02:34:15,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 02:34:15,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 02:34:15,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 02:34:15,979 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 02:34:15,979 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 02:34:15,982 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 02:34:15,982 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-16 02:34:15,983 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 02:34:15,983 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-16T02:34:16,205 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2 2024-11-16 02:34:16,209 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 02:34:16,209 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T02:34:16,217 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-16T02:34:16,225 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-16T02:34:16,254 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T02:34:16,315 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T02:34:16,316 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T02:34:16,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T02:34:16,350 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39, deleteOnExit=true 2024-11-16T02:34:16,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T02:34:16,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/test.cache.data in system properties and HBase conf 2024-11-16T02:34:16,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T02:34:16,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/hadoop.log.dir in system properties and HBase conf 2024-11-16T02:34:16,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T02:34:16,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T02:34:16,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T02:34:16,434 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T02:34:16,527 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T02:34:16,531 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T02:34:16,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T02:34:16,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T02:34:16,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T02:34:16,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T02:34:16,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T02:34:16,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T02:34:16,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T02:34:16,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T02:34:16,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/nfs.dump.dir in system properties and HBase conf 2024-11-16T02:34:16,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/java.io.tmpdir in system properties and HBase conf 2024-11-16T02:34:16,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T02:34:16,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T02:34:16,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T02:34:17,566 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T02:34:17,638 INFO [Time-limited test {}] log.Log(170): Logging initialized @2456ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T02:34:17,703 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T02:34:17,765 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T02:34:17,792 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T02:34:17,792 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T02:34:17,794 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T02:34:17,806 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T02:34:17,809 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/hadoop.log.dir/,AVAILABLE} 2024-11-16T02:34:17,810 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T02:34:17,983 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/java.io.tmpdir/jetty-localhost-42843-hadoop-hdfs-3_4_1-tests_jar-_-any-14322323960158748755/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T02:34:17,989 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:42843} 2024-11-16T02:34:17,989 INFO [Time-limited test {}] server.Server(415): Started @2807ms 2024-11-16T02:34:18,529 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T02:34:18,537 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T02:34:18,538 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T02:34:18,538 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T02:34:18,538 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T02:34:18,539 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/hadoop.log.dir/,AVAILABLE} 2024-11-16T02:34:18,539 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T02:34:18,636 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/java.io.tmpdir/jetty-localhost-45243-hadoop-hdfs-3_4_1-tests_jar-_-any-5284707671007427083/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T02:34:18,637 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:45243} 2024-11-16T02:34:18,637 INFO [Time-limited test {}] server.Server(415): Started @3455ms 2024-11-16T02:34:18,681 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T02:34:18,783 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T02:34:18,788 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T02:34:18,796 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T02:34:18,797 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T02:34:18,797 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T02:34:18,798 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/hadoop.log.dir/,AVAILABLE} 2024-11-16T02:34:18,800 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T02:34:18,941 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/java.io.tmpdir/jetty-localhost-34499-hadoop-hdfs-3_4_1-tests_jar-_-any-7101043238026675890/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T02:34:18,942 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:34499} 2024-11-16T02:34:18,943 INFO [Time-limited test {}] server.Server(415): Started @3760ms 2024-11-16T02:34:18,945 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T02:34:18,978 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T02:34:18,983 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T02:34:18,987 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T02:34:18,987 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T02:34:18,987 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T02:34:18,988 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/hadoop.log.dir/,AVAILABLE} 2024-11-16T02:34:18,989 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T02:34:19,083 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/java.io.tmpdir/jetty-localhost-35589-hadoop-hdfs-3_4_1-tests_jar-_-any-1707585037425322081/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T02:34:19,084 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:35589} 2024-11-16T02:34:19,084 INFO [Time-limited test {}] server.Server(415): Started @3902ms 2024-11-16T02:34:19,086 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T02:34:20,524 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data1/current/BP-1221085024-172.17.0.3-1731724457043/current, will proceed with Du for space computation calculation, 2024-11-16T02:34:20,524 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data2/current/BP-1221085024-172.17.0.3-1731724457043/current, will proceed with Du for space computation calculation, 2024-11-16T02:34:20,549 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T02:34:20,594 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbd53896211971361 with lease ID 0xfaff8ed0e28cd134: Processing first storage report for DS-eabca314-d433-4de7-9a38-1d7d16e3081f from datanode DatanodeRegistration(127.0.0.1:33993, datanodeUuid=ba902d54-5219-4efc-bd09-dd1188434d48, infoPort=41131, infoSecurePort=0, ipcPort=33733, storageInfo=lv=-57;cid=testClusterID;nsid=293219826;c=1731724457043) 2024-11-16T02:34:20,595 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbd53896211971361 with lease ID 0xfaff8ed0e28cd134: from storage DS-eabca314-d433-4de7-9a38-1d7d16e3081f node DatanodeRegistration(127.0.0.1:33993, datanodeUuid=ba902d54-5219-4efc-bd09-dd1188434d48, infoPort=41131, infoSecurePort=0, ipcPort=33733, storageInfo=lv=-57;cid=testClusterID;nsid=293219826;c=1731724457043), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-16T02:34:20,596 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbd53896211971361 with lease ID 0xfaff8ed0e28cd134: Processing first storage report for DS-572f8a01-78e5-4c98-ab95-a4767c8eeb2f from datanode DatanodeRegistration(127.0.0.1:33993, datanodeUuid=ba902d54-5219-4efc-bd09-dd1188434d48, infoPort=41131, infoSecurePort=0, ipcPort=33733, storageInfo=lv=-57;cid=testClusterID;nsid=293219826;c=1731724457043) 2024-11-16T02:34:20,596 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbd53896211971361 with lease ID 0xfaff8ed0e28cd134: from storage DS-572f8a01-78e5-4c98-ab95-a4767c8eeb2f node DatanodeRegistration(127.0.0.1:33993, datanodeUuid=ba902d54-5219-4efc-bd09-dd1188434d48, infoPort=41131, infoSecurePort=0, ipcPort=33733, storageInfo=lv=-57;cid=testClusterID;nsid=293219826;c=1731724457043), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T02:34:20,611 WARN [Thread-134 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data4/current/BP-1221085024-172.17.0.3-1731724457043/current, will proceed with Du for space computation calculation, 2024-11-16T02:34:20,611 WARN [Thread-133 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data3/current/BP-1221085024-172.17.0.3-1731724457043/current, will proceed with Du for space computation calculation, 2024-11-16T02:34:20,612 WARN [Thread-135 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data5/current/BP-1221085024-172.17.0.3-1731724457043/current, will proceed with Du for space computation calculation, 2024-11-16T02:34:20,612 WARN [Thread-136 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data6/current/BP-1221085024-172.17.0.3-1731724457043/current, will proceed with Du for space computation calculation, 2024-11-16T02:34:20,635 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T02:34:20,635 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T02:34:20,639 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x318bc2462a1046db with lease ID 0xfaff8ed0e28cd135: Processing first storage report for DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b from datanode DatanodeRegistration(127.0.0.1:40973, datanodeUuid=39717887-3138-4201-8842-0eece2fe6967, infoPort=39667, infoSecurePort=0, ipcPort=46059, storageInfo=lv=-57;cid=testClusterID;nsid=293219826;c=1731724457043) 2024-11-16T02:34:20,640 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x318bc2462a1046db with lease ID 0xfaff8ed0e28cd135: from storage DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b node DatanodeRegistration(127.0.0.1:40973, datanodeUuid=39717887-3138-4201-8842-0eece2fe6967, infoPort=39667, infoSecurePort=0, ipcPort=46059, storageInfo=lv=-57;cid=testClusterID;nsid=293219826;c=1731724457043), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T02:34:20,640 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x28cf83927f4ed189 with lease ID 0xfaff8ed0e28cd136: Processing first storage report for DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586 from datanode DatanodeRegistration(127.0.0.1:39531, datanodeUuid=3f36a101-aafe-43cc-8014-0d1f212b982f, infoPort=38477, infoSecurePort=0, ipcPort=40745, storageInfo=lv=-57;cid=testClusterID;nsid=293219826;c=1731724457043) 2024-11-16T02:34:20,640 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x28cf83927f4ed189 with lease ID 0xfaff8ed0e28cd136: from storage DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586 node DatanodeRegistration(127.0.0.1:39531, datanodeUuid=3f36a101-aafe-43cc-8014-0d1f212b982f, infoPort=38477, infoSecurePort=0, ipcPort=40745, storageInfo=lv=-57;cid=testClusterID;nsid=293219826;c=1731724457043), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T02:34:20,641 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x318bc2462a1046db with lease ID 0xfaff8ed0e28cd135: Processing first storage report for DS-6d964398-bd4b-493a-9f6c-3803aaa74735 from datanode DatanodeRegistration(127.0.0.1:40973, datanodeUuid=39717887-3138-4201-8842-0eece2fe6967, infoPort=39667, infoSecurePort=0, ipcPort=46059, storageInfo=lv=-57;cid=testClusterID;nsid=293219826;c=1731724457043) 2024-11-16T02:34:20,641 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x318bc2462a1046db with lease ID 0xfaff8ed0e28cd135: from storage DS-6d964398-bd4b-493a-9f6c-3803aaa74735 node DatanodeRegistration(127.0.0.1:40973, datanodeUuid=39717887-3138-4201-8842-0eece2fe6967, infoPort=39667, infoSecurePort=0, ipcPort=46059, storageInfo=lv=-57;cid=testClusterID;nsid=293219826;c=1731724457043), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T02:34:20,641 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x28cf83927f4ed189 with lease ID 0xfaff8ed0e28cd136: Processing first storage report for DS-6324e411-e84a-456c-8099-37c2699268a7 from datanode DatanodeRegistration(127.0.0.1:39531, datanodeUuid=3f36a101-aafe-43cc-8014-0d1f212b982f, infoPort=38477, infoSecurePort=0, ipcPort=40745, storageInfo=lv=-57;cid=testClusterID;nsid=293219826;c=1731724457043) 2024-11-16T02:34:20,641 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x28cf83927f4ed189 with lease ID 0xfaff8ed0e28cd136: from storage DS-6324e411-e84a-456c-8099-37c2699268a7 node DatanodeRegistration(127.0.0.1:39531, datanodeUuid=3f36a101-aafe-43cc-8014-0d1f212b982f, infoPort=38477, infoSecurePort=0, ipcPort=40745, storageInfo=lv=-57;cid=testClusterID;nsid=293219826;c=1731724457043), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T02:34:20,714 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2 2024-11-16T02:34:20,784 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/zookeeper_0, clientPort=51370, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T02:34:20,793 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51370 2024-11-16T02:34:20,802 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T02:34:20,804 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T02:34:21,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741825_1001 (size=7) 2024-11-16T02:34:21,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741825_1001 (size=7) 2024-11-16T02:34:21,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741825_1001 (size=7) 2024-11-16T02:34:21,415 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9 with version=8 2024-11-16T02:34:21,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/hbase-staging 2024-11-16T02:34:21,704 INFO [Time-limited test {}] client.ConnectionUtils(128): master/667222f7cf1b:0 server-side Connection retries=45 2024-11-16T02:34:21,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T02:34:21,713 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T02:34:21,717 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T02:34:21,717 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T02:34:21,718 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T02:34:21,876 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T02:34:21,933 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T02:34:21,941 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T02:34:21,945 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T02:34:21,967 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 95823 (auto-detected) 2024-11-16T02:34:21,968 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-16T02:34:21,985 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38401 2024-11-16T02:34:22,006 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38401 connecting to ZooKeeper ensemble=127.0.0.1:51370 2024-11-16T02:34:22,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:384010x0, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T02:34:22,118 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38401-0x101417e3e890000 connected 2024-11-16T02:34:22,214 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T02:34:22,220 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T02:34:22,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T02:34:22,236 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9, hbase.cluster.distributed=false 2024-11-16T02:34:22,256 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T02:34:22,260 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38401 2024-11-16T02:34:22,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38401 2024-11-16T02:34:22,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38401 2024-11-16T02:34:22,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38401 2024-11-16T02:34:22,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38401 2024-11-16T02:34:22,352 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/667222f7cf1b:0 server-side Connection retries=45 2024-11-16T02:34:22,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T02:34:22,354 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T02:34:22,354 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T02:34:22,354 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T02:34:22,354 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T02:34:22,356 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T02:34:22,359 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T02:34:22,359 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40547 2024-11-16T02:34:22,361 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40547 connecting to ZooKeeper ensemble=127.0.0.1:51370 2024-11-16T02:34:22,362 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T02:34:22,365 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T02:34:22,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405470x0, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T02:34:22,381 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:405470x0, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T02:34:22,381 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40547-0x101417e3e890001 connected 2024-11-16T02:34:22,385 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T02:34:22,391 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T02:34:22,394 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T02:34:22,398 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T02:34:22,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40547 2024-11-16T02:34:22,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40547 2024-11-16T02:34:22,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40547 2024-11-16T02:34:22,403 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40547 2024-11-16T02:34:22,403 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40547 2024-11-16T02:34:22,419 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/667222f7cf1b:0 server-side Connection retries=45 2024-11-16T02:34:22,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T02:34:22,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T02:34:22,419 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T02:34:22,420 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T02:34:22,420 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T02:34:22,420 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T02:34:22,420 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T02:34:22,421 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40209 2024-11-16T02:34:22,424 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40209 connecting to ZooKeeper ensemble=127.0.0.1:51370 2024-11-16T02:34:22,425 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T02:34:22,428 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T02:34:22,444 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:402090x0, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T02:34:22,444 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T02:34:22,444 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40209-0x101417e3e890002 connected 2024-11-16T02:34:22,445 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T02:34:22,446 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T02:34:22,447 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T02:34:22,449 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T02:34:22,450 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40209 2024-11-16T02:34:22,450 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40209 2024-11-16T02:34:22,450 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40209 2024-11-16T02:34:22,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40209 2024-11-16T02:34:22,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40209 2024-11-16T02:34:22,468 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/667222f7cf1b:0 server-side Connection retries=45 2024-11-16T02:34:22,468 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T02:34:22,468 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T02:34:22,468 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T02:34:22,469 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T02:34:22,469 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T02:34:22,469 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T02:34:22,469 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T02:34:22,470 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39421 2024-11-16T02:34:22,472 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39421 connecting to ZooKeeper ensemble=127.0.0.1:51370 2024-11-16T02:34:22,473 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T02:34:22,476 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T02:34:22,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:394210x0, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T02:34:22,486 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:394210x0, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T02:34:22,486 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39421-0x101417e3e890003 connected 2024-11-16T02:34:22,487 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T02:34:22,488 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T02:34:22,489 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T02:34:22,491 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T02:34:22,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39421 2024-11-16T02:34:22,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39421 2024-11-16T02:34:22,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39421 2024-11-16T02:34:22,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39421 2024-11-16T02:34:22,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39421 2024-11-16T02:34:22,509 DEBUG [M:0;667222f7cf1b:38401 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;667222f7cf1b:38401 2024-11-16T02:34:22,510 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/667222f7cf1b,38401,1731724461545 2024-11-16T02:34:22,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T02:34:22,528 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T02:34:22,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T02:34:22,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T02:34:22,530 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/667222f7cf1b,38401,1731724461545 2024-11-16T02:34:22,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T02:34:22,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:22,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T02:34:22,559 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T02:34:22,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:22,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:22,559 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:22,560 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T02:34:22,561 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/667222f7cf1b,38401,1731724461545 from backup master directory 2024-11-16T02:34:22,570 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T02:34:22,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T02:34:22,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T02:34:22,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/667222f7cf1b,38401,1731724461545 2024-11-16T02:34:22,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T02:34:22,572 WARN [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T02:34:22,572 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=667222f7cf1b,38401,1731724461545 2024-11-16T02:34:22,575 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T02:34:22,577 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T02:34:22,636 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/hbase.id] with ID: 5c137c35-6caf-4e9b-872c-afcb989f7817 2024-11-16T02:34:22,636 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/.tmp/hbase.id 2024-11-16T02:34:22,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741826_1002 (size=42) 2024-11-16T02:34:22,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741826_1002 (size=42) 2024-11-16T02:34:22,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741826_1002 (size=42) 2024-11-16T02:34:22,652 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/.tmp/hbase.id]:[hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/hbase.id] 2024-11-16T02:34:22,700 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T02:34:22,704 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T02:34:22,723 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-16T02:34:22,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:22,738 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:22,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:22,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:22,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741827_1003 (size=196) 2024-11-16T02:34:22,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741827_1003 (size=196) 2024-11-16T02:34:22,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741827_1003 (size=196) 2024-11-16T02:34:22,776 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T02:34:22,777 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T02:34:22,782 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:22,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741828_1004 (size=1189) 2024-11-16T02:34:22,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741828_1004 (size=1189) 2024-11-16T02:34:22,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741828_1004 (size=1189) 2024-11-16T02:34:22,831 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store 2024-11-16T02:34:22,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741829_1005 (size=34) 2024-11-16T02:34:22,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741829_1005 (size=34) 2024-11-16T02:34:22,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741829_1005 (size=34) 2024-11-16T02:34:22,856 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T02:34:22,861 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:22,863 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T02:34:22,863 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T02:34:22,863 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T02:34:22,865 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T02:34:22,865 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T02:34:22,865 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T02:34:22,867 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731724462862Disabling compacts and flushes for region at 1731724462862Disabling writes for close at 1731724462865 (+3 ms)Writing region close event to WAL at 1731724462865Closed at 1731724462865 2024-11-16T02:34:22,869 WARN [master/667222f7cf1b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/.initializing 2024-11-16T02:34:22,869 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/WALs/667222f7cf1b,38401,1731724461545 2024-11-16T02:34:22,877 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:22,893 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=667222f7cf1b%2C38401%2C1731724461545, suffix=, logDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/WALs/667222f7cf1b,38401,1731724461545, archiveDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/oldWALs, maxLogs=10 2024-11-16T02:34:22,919 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/WALs/667222f7cf1b,38401,1731724461545/667222f7cf1b%2C38401%2C1731724461545.1731724462897, exclude list is [], retry=0 2024-11-16T02:34:22,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:22,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:22,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:22,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:22,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T02:34:22,973 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-16T02:34:23,029 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/WALs/667222f7cf1b,38401,1731724461545/667222f7cf1b%2C38401%2C1731724461545.1731724462897 2024-11-16T02:34:23,030 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477)] 2024-11-16T02:34:23,031 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:23,031 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:23,034 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T02:34:23,034 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T02:34:23,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T02:34:23,092 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T02:34:23,095 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:23,097 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T02:34:23,098 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T02:34:23,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T02:34:23,101 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:23,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:23,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T02:34:23,105 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T02:34:23,105 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:23,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:23,107 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T02:34:23,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T02:34:23,110 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:23,111 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:23,112 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T02:34:23,115 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T02:34:23,116 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T02:34:23,121 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T02:34:23,121 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T02:34:23,124 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T02:34:23,127 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T02:34:23,132 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:23,133 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60134595, jitterRate=-0.1039247065782547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T02:34:23,140 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731724463046Initializing all the Stores at 1731724463048 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731724463048Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724463049 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724463049Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724463049Cleaning up temporary data from old regions at 1731724463121 (+72 ms)Region opened successfully at 1731724463140 (+19 ms) 2024-11-16T02:34:23,141 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T02:34:23,172 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ba1c860, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=667222f7cf1b/172.17.0.3:0 2024-11-16T02:34:23,198 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T02:34:23,208 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T02:34:23,208 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T02:34:23,210 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T02:34:23,212 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T02:34:23,216 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T02:34:23,216 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T02:34:23,240 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T02:34:23,249 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T02:34:23,304 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T02:34:23,307 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T02:34:23,309 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T02:34:23,317 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T02:34:23,319 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T02:34:23,323 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T02:34:23,335 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T02:34:23,337 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T02:34:23,348 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T02:34:23,371 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T02:34:23,380 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T02:34:23,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T02:34:23,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T02:34:23,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T02:34:23,391 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T02:34:23,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:23,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:23,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:23,391 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:23,395 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=667222f7cf1b,38401,1731724461545, sessionid=0x101417e3e890000, setting cluster-up flag (Was=false) 2024-11-16T02:34:23,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:23,422 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:23,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:23,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:23,454 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T02:34:23,458 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=667222f7cf1b,38401,1731724461545 2024-11-16T02:34:23,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:23,485 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:23,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:23,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:23,517 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T02:34:23,519 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=667222f7cf1b,38401,1731724461545 2024-11-16T02:34:23,525 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T02:34:23,592 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T02:34:23,599 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(746): ClusterId : 5c137c35-6caf-4e9b-872c-afcb989f7817 2024-11-16T02:34:23,599 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(746): ClusterId : 5c137c35-6caf-4e9b-872c-afcb989f7817 2024-11-16T02:34:23,599 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(746): ClusterId : 5c137c35-6caf-4e9b-872c-afcb989f7817 2024-11-16T02:34:23,602 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T02:34:23,602 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T02:34:23,602 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T02:34:23,602 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T02:34:23,608 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T02:34:23,613 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 667222f7cf1b,38401,1731724461545 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T02:34:23,624 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T02:34:23,624 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T02:34:23,624 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T02:34:23,624 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T02:34:23,624 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T02:34:23,624 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T02:34:23,625 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/667222f7cf1b:0, corePoolSize=5, maxPoolSize=5 2024-11-16T02:34:23,625 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/667222f7cf1b:0, corePoolSize=5, maxPoolSize=5 2024-11-16T02:34:23,625 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/667222f7cf1b:0, corePoolSize=5, maxPoolSize=5 2024-11-16T02:34:23,625 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/667222f7cf1b:0, corePoolSize=5, maxPoolSize=5 2024-11-16T02:34:23,625 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/667222f7cf1b:0, corePoolSize=10, maxPoolSize=10 2024-11-16T02:34:23,625 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,626 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/667222f7cf1b:0, corePoolSize=2, maxPoolSize=2 2024-11-16T02:34:23,626 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,628 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731724493628 2024-11-16T02:34:23,629 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T02:34:23,630 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T02:34:23,631 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T02:34:23,631 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T02:34:23,634 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T02:34:23,634 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T02:34:23,634 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T02:34:23,634 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T02:34:23,634 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T02:34:23,635 DEBUG [RS:2;667222f7cf1b:39421 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c8a45d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=667222f7cf1b/172.17.0.3:0 2024-11-16T02:34:23,635 DEBUG [RS:0;667222f7cf1b:40547 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33754245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=667222f7cf1b/172.17.0.3:0 2024-11-16T02:34:23,635 DEBUG [RS:1;667222f7cf1b:40209 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f2acd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=667222f7cf1b/172.17.0.3:0 2024-11-16T02:34:23,635 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T02:34:23,635 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T02:34:23,636 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,638 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:23,638 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T02:34:23,640 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T02:34:23,641 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T02:34:23,642 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T02:34:23,644 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T02:34:23,644 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T02:34:23,650 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/667222f7cf1b:0:becomeActiveMaster-HFileCleaner.large.0-1731724463645,5,FailOnTimeoutGroup] 2024-11-16T02:34:23,651 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/667222f7cf1b:0:becomeActiveMaster-HFileCleaner.small.0-1731724463651,5,FailOnTimeoutGroup] 2024-11-16T02:34:23,651 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,652 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T02:34:23,652 DEBUG [RS:0;667222f7cf1b:40547 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;667222f7cf1b:40547 2024-11-16T02:34:23,653 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,653 DEBUG [RS:1;667222f7cf1b:40209 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;667222f7cf1b:40209 2024-11-16T02:34:23,653 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,655 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T02:34:23,655 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T02:34:23,656 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T02:34:23,656 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T02:34:23,656 DEBUG [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T02:34:23,656 DEBUG [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T02:34:23,656 DEBUG [RS:2;667222f7cf1b:39421 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;667222f7cf1b:39421 2024-11-16T02:34:23,656 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T02:34:23,656 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T02:34:23,656 DEBUG [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T02:34:23,659 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(2659): reportForDuty to master=667222f7cf1b,38401,1731724461545 with port=39421, startcode=1731724462467 2024-11-16T02:34:23,659 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(2659): reportForDuty to master=667222f7cf1b,38401,1731724461545 with port=40209, startcode=1731724462418 2024-11-16T02:34:23,659 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(2659): reportForDuty to master=667222f7cf1b,38401,1731724461545 with port=40547, startcode=1731724462323 2024-11-16T02:34:23,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741831_1007 (size=1321) 2024-11-16T02:34:23,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741831_1007 (size=1321) 2024-11-16T02:34:23,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741831_1007 (size=1321) 2024-11-16T02:34:23,664 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T02:34:23,665 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9 2024-11-16T02:34:23,672 DEBUG [RS:1;667222f7cf1b:40209 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T02:34:23,672 DEBUG [RS:2;667222f7cf1b:39421 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T02:34:23,672 DEBUG [RS:0;667222f7cf1b:40547 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T02:34:23,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741832_1008 (size=32) 2024-11-16T02:34:23,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741832_1008 (size=32) 2024-11-16T02:34:23,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741832_1008 (size=32) 2024-11-16T02:34:23,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:23,691 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T02:34:23,694 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T02:34:23,694 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:23,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T02:34:23,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T02:34:23,697 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T02:34:23,697 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:23,698 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T02:34:23,698 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T02:34:23,701 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T02:34:23,701 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:23,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T02:34:23,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T02:34:23,704 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T02:34:23,704 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:23,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T02:34:23,705 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T02:34:23,706 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740 2024-11-16T02:34:23,707 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740 2024-11-16T02:34:23,710 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T02:34:23,710 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T02:34:23,711 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T02:34:23,713 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T02:34:23,719 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:23,721 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71918801, jitterRate=0.07167364656925201}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T02:34:23,725 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731724463689Initializing all the Stores at 1731724463691 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731724463691Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731724463691Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724463691Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731724463691Cleaning up temporary data from old regions at 1731724463710 (+19 ms)Region opened successfully at 1731724463725 (+15 ms) 2024-11-16T02:34:23,725 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T02:34:23,726 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T02:34:23,726 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T02:34:23,726 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T02:34:23,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T02:34:23,728 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T02:34:23,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731724463725Disabling compacts and flushes for region at 1731724463725Disabling writes for close at 1731724463726 (+1 ms)Writing region close event to WAL at 1731724463728 (+2 ms)Closed at 1731724463728 2024-11-16T02:34:23,733 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T02:34:23,733 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T02:34:23,737 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55333, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T02:34:23,737 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40233, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T02:34:23,737 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36277, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T02:34:23,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T02:34:23,745 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38401 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 667222f7cf1b,40547,1731724462323 2024-11-16T02:34:23,748 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38401 {}] master.ServerManager(517): Registering regionserver=667222f7cf1b,40547,1731724462323 2024-11-16T02:34:23,755 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T02:34:23,759 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T02:34:23,763 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38401 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 667222f7cf1b,40209,1731724462418 2024-11-16T02:34:23,763 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38401 {}] master.ServerManager(517): Registering regionserver=667222f7cf1b,40209,1731724462418 2024-11-16T02:34:23,767 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38401 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 667222f7cf1b,39421,1731724462467 2024-11-16T02:34:23,767 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38401 {}] master.ServerManager(517): Registering regionserver=667222f7cf1b,39421,1731724462467 2024-11-16T02:34:23,767 DEBUG [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9 2024-11-16T02:34:23,767 DEBUG [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43879 2024-11-16T02:34:23,767 DEBUG [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T02:34:23,770 DEBUG [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9 2024-11-16T02:34:23,770 DEBUG [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43879 2024-11-16T02:34:23,770 DEBUG [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T02:34:23,771 DEBUG [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9 2024-11-16T02:34:23,771 DEBUG [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43879 2024-11-16T02:34:23,772 DEBUG [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T02:34:23,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T02:34:23,821 DEBUG [RS:1;667222f7cf1b:40209 {}] zookeeper.ZKUtil(111): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/667222f7cf1b,40209,1731724462418 2024-11-16T02:34:23,821 WARN [RS:1;667222f7cf1b:40209 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T02:34:23,822 INFO [RS:1;667222f7cf1b:40209 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:23,822 DEBUG [RS:2;667222f7cf1b:39421 {}] zookeeper.ZKUtil(111): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/667222f7cf1b,39421,1731724462467 2024-11-16T02:34:23,822 WARN [RS:2;667222f7cf1b:39421 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T02:34:23,822 DEBUG [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418 2024-11-16T02:34:23,822 INFO [RS:2;667222f7cf1b:39421 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:23,822 DEBUG [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,39421,1731724462467 2024-11-16T02:34:23,822 DEBUG [RS:0;667222f7cf1b:40547 {}] zookeeper.ZKUtil(111): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/667222f7cf1b,40547,1731724462323 2024-11-16T02:34:23,822 WARN [RS:0;667222f7cf1b:40547 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T02:34:23,823 INFO [RS:0;667222f7cf1b:40547 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:23,823 DEBUG [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40547,1731724462323 2024-11-16T02:34:23,823 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [667222f7cf1b,40209,1731724462418] 2024-11-16T02:34:23,823 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [667222f7cf1b,39421,1731724462467] 2024-11-16T02:34:23,823 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [667222f7cf1b,40547,1731724462323] 2024-11-16T02:34:23,848 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T02:34:23,848 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T02:34:23,848 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T02:34:23,865 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T02:34:23,865 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T02:34:23,865 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T02:34:23,871 INFO [RS:1;667222f7cf1b:40209 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T02:34:23,871 INFO [RS:2;667222f7cf1b:39421 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T02:34:23,871 INFO [RS:0;667222f7cf1b:40547 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T02:34:23,871 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,871 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,871 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,876 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T02:34:23,876 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T02:34:23,876 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T02:34:23,883 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T02:34:23,883 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T02:34:23,883 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T02:34:23,885 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,885 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,885 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,885 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,885 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,885 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,885 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,885 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,885 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0, corePoolSize=2, maxPoolSize=2 2024-11-16T02:34:23,886 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0, corePoolSize=2, maxPoolSize=2 2024-11-16T02:34:23,886 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0, corePoolSize=2, maxPoolSize=2 2024-11-16T02:34:23,886 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,886 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/667222f7cf1b:0, corePoolSize=1, maxPoolSize=1 2024-11-16T02:34:23,887 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/667222f7cf1b:0, corePoolSize=3, maxPoolSize=3 2024-11-16T02:34:23,887 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/667222f7cf1b:0, corePoolSize=3, maxPoolSize=3 2024-11-16T02:34:23,887 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/667222f7cf1b:0, corePoolSize=3, maxPoolSize=3 2024-11-16T02:34:23,887 DEBUG [RS:0;667222f7cf1b:40547 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/667222f7cf1b:0, corePoolSize=3, maxPoolSize=3 2024-11-16T02:34:23,887 DEBUG [RS:2;667222f7cf1b:39421 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/667222f7cf1b:0, corePoolSize=3, maxPoolSize=3 2024-11-16T02:34:23,887 DEBUG [RS:1;667222f7cf1b:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/667222f7cf1b:0, corePoolSize=3, maxPoolSize=3 2024-11-16T02:34:23,892 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,893 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,893 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,893 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,893 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,893 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,893 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,893 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,893 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,893 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ChoreService(168): Chore ScheduledChore name=667222f7cf1b,39421,1731724462467-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T02:34:23,893 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,893 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=667222f7cf1b,40209,1731724462418-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T02:34:23,896 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,896 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,896 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,896 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,896 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,897 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ChoreService(168): Chore ScheduledChore name=667222f7cf1b,40547,1731724462323-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T02:34:23,910 WARN [667222f7cf1b:38401 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-16T02:34:23,918 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T02:34:23,920 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=667222f7cf1b,40209,1731724462418-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,920 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,920 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.Replication(171): 667222f7cf1b,40209,1731724462418 started 2024-11-16T02:34:23,922 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T02:34:23,922 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T02:34:23,922 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ChoreService(168): Chore ScheduledChore name=667222f7cf1b,39421,1731724462467-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,922 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ChoreService(168): Chore ScheduledChore name=667222f7cf1b,40547,1731724462323-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,922 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,922 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,922 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.Replication(171): 667222f7cf1b,39421,1731724462467 started 2024-11-16T02:34:23,922 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.Replication(171): 667222f7cf1b,40547,1731724462323 started 2024-11-16T02:34:23,939 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,940 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(1482): Serving as 667222f7cf1b,40209,1731724462418, RpcServer on 667222f7cf1b/172.17.0.3:40209, sessionid=0x101417e3e890002 2024-11-16T02:34:23,941 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T02:34:23,941 DEBUG [RS:1;667222f7cf1b:40209 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 667222f7cf1b,40209,1731724462418 2024-11-16T02:34:23,941 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '667222f7cf1b,40209,1731724462418' 2024-11-16T02:34:23,941 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T02:34:23,942 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T02:34:23,943 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T02:34:23,943 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T02:34:23,943 DEBUG [RS:1;667222f7cf1b:40209 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 667222f7cf1b,40209,1731724462418 2024-11-16T02:34:23,943 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '667222f7cf1b,40209,1731724462418' 2024-11-16T02:34:23,943 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T02:34:23,944 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T02:34:23,944 DEBUG [RS:1;667222f7cf1b:40209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T02:34:23,945 INFO [RS:1;667222f7cf1b:40209 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T02:34:23,945 INFO [RS:1;667222f7cf1b:40209 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T02:34:23,945 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,945 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:23,945 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(1482): Serving as 667222f7cf1b,40547,1731724462323, RpcServer on 667222f7cf1b/172.17.0.3:40547, sessionid=0x101417e3e890001 2024-11-16T02:34:23,945 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(1482): Serving as 667222f7cf1b,39421,1731724462467, RpcServer on 667222f7cf1b/172.17.0.3:39421, sessionid=0x101417e3e890003 2024-11-16T02:34:23,946 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T02:34:23,946 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T02:34:23,946 DEBUG [RS:2;667222f7cf1b:39421 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 667222f7cf1b,39421,1731724462467 2024-11-16T02:34:23,946 DEBUG [RS:0;667222f7cf1b:40547 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 667222f7cf1b,40547,1731724462323 2024-11-16T02:34:23,946 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '667222f7cf1b,39421,1731724462467' 2024-11-16T02:34:23,946 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '667222f7cf1b,40547,1731724462323' 2024-11-16T02:34:23,946 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T02:34:23,946 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T02:34:23,947 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T02:34:23,947 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T02:34:23,947 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T02:34:23,947 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T02:34:23,947 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T02:34:23,947 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T02:34:23,947 DEBUG [RS:2;667222f7cf1b:39421 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 667222f7cf1b,39421,1731724462467 2024-11-16T02:34:23,948 DEBUG [RS:0;667222f7cf1b:40547 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 667222f7cf1b,40547,1731724462323 2024-11-16T02:34:23,948 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '667222f7cf1b,39421,1731724462467' 2024-11-16T02:34:23,948 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T02:34:23,948 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '667222f7cf1b,40547,1731724462323' 2024-11-16T02:34:23,948 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T02:34:23,948 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T02:34:23,948 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T02:34:23,949 DEBUG [RS:2;667222f7cf1b:39421 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T02:34:23,949 DEBUG [RS:0;667222f7cf1b:40547 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T02:34:23,949 INFO [RS:2;667222f7cf1b:39421 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T02:34:23,949 INFO [RS:0;667222f7cf1b:40547 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T02:34:23,949 INFO [RS:2;667222f7cf1b:39421 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T02:34:23,949 INFO [RS:0;667222f7cf1b:40547 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T02:34:24,049 INFO [RS:0;667222f7cf1b:40547 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:24,049 INFO [RS:2;667222f7cf1b:39421 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:24,049 INFO [RS:1;667222f7cf1b:40209 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:24,053 INFO [RS:0;667222f7cf1b:40547 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=667222f7cf1b%2C40547%2C1731724462323, suffix=, logDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40547,1731724462323, archiveDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/oldWALs, maxLogs=32 2024-11-16T02:34:24,053 INFO [RS:2;667222f7cf1b:39421 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=667222f7cf1b%2C39421%2C1731724462467, suffix=, logDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,39421,1731724462467, archiveDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/oldWALs, maxLogs=32 2024-11-16T02:34:24,054 INFO [RS:1;667222f7cf1b:40209 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=667222f7cf1b%2C40209%2C1731724462418, suffix=, logDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418, archiveDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/oldWALs, maxLogs=32 2024-11-16T02:34:24,070 DEBUG [RS:2;667222f7cf1b:39421 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,39421,1731724462467/667222f7cf1b%2C39421%2C1731724462467.1731724464057, exclude list is [], retry=0 2024-11-16T02:34:24,070 DEBUG [RS:0;667222f7cf1b:40547 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40547,1731724462323/667222f7cf1b%2C40547%2C1731724462323.1731724464057, exclude list is [], retry=0 2024-11-16T02:34:24,070 DEBUG [RS:1;667222f7cf1b:40209 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418/667222f7cf1b%2C40209%2C1731724462418.1731724464057, exclude list is [], retry=0 2024-11-16T02:34:24,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:24,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:24,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:24,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:24,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:24,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:24,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:24,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:24,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:24,108 INFO [RS:1;667222f7cf1b:40209 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418/667222f7cf1b%2C40209%2C1731724462418.1731724464057 2024-11-16T02:34:24,109 DEBUG [RS:1;667222f7cf1b:40209 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:24,121 INFO [RS:0;667222f7cf1b:40547 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40547,1731724462323/667222f7cf1b%2C40547%2C1731724462323.1731724464057 2024-11-16T02:34:24,121 DEBUG [RS:0;667222f7cf1b:40547 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:24,122 INFO [RS:2;667222f7cf1b:39421 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,39421,1731724462467/667222f7cf1b%2C39421%2C1731724462467.1731724464057 2024-11-16T02:34:24,124 DEBUG [RS:2;667222f7cf1b:39421 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:41131:41131)] 2024-11-16T02:34:24,162 DEBUG [667222f7cf1b:38401 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-16T02:34:24,169 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(204): Hosts are {667222f7cf1b=0} racks are {/default-rack=0} 2024-11-16T02:34:24,176 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T02:34:24,177 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T02:34:24,177 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-16T02:34:24,177 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T02:34:24,177 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T02:34:24,177 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T02:34:24,177 INFO [667222f7cf1b:38401 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T02:34:24,177 INFO [667222f7cf1b:38401 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T02:34:24,177 INFO [667222f7cf1b:38401 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T02:34:24,177 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T02:34:24,185 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=667222f7cf1b,40547,1731724462323 2024-11-16T02:34:24,190 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 667222f7cf1b,40547,1731724462323, state=OPENING 2024-11-16T02:34:24,241 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T02:34:24,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:24,254 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:24,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:24,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:34:24,254 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T02:34:24,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T02:34:24,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T02:34:24,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T02:34:24,256 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T02:34:24,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=667222f7cf1b,40547,1731724462323}] 2024-11-16T02:34:24,440 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T02:34:24,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58269, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T02:34:24,455 INFO [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T02:34:24,455 INFO [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:24,456 INFO [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T02:34:24,459 INFO [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=667222f7cf1b%2C40547%2C1731724462323.meta, suffix=.meta, logDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40547,1731724462323, archiveDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/oldWALs, maxLogs=32 2024-11-16T02:34:24,475 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40547,1731724462323/667222f7cf1b%2C40547%2C1731724462323.meta.1731724464462.meta, exclude list is [], retry=0 2024-11-16T02:34:24,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:24,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:24,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:24,511 INFO [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40547,1731724462323/667222f7cf1b%2C40547%2C1731724462323.meta.1731724464462.meta 2024-11-16T02:34:24,512 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:38477:38477)] 2024-11-16T02:34:24,512 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:24,514 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T02:34:24,517 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T02:34:24,522 INFO [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T02:34:24,527 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T02:34:24,527 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:24,528 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T02:34:24,528 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T02:34:24,531 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T02:34:24,533 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T02:34:24,533 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:24,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T02:34:24,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T02:34:24,535 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T02:34:24,535 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:24,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T02:34:24,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T02:34:24,538 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T02:34:24,538 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:24,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T02:34:24,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T02:34:24,540 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T02:34:24,540 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:24,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T02:34:24,541 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T02:34:24,543 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740 2024-11-16T02:34:24,545 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740 2024-11-16T02:34:24,548 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T02:34:24,548 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T02:34:24,549 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T02:34:24,551 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T02:34:24,554 INFO [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63311110, jitterRate=-0.05659094452857971}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T02:34:24,554 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T02:34:24,555 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731724464528Writing region info on filesystem at 1731724464529 (+1 ms)Initializing all the Stores at 1731724464531 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731724464531Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731724464531Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724464531Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731724464531Cleaning up temporary data from old regions at 1731724464548 (+17 ms)Running coprocessor post-open hooks at 1731724464554 (+6 ms)Region opened successfully at 1731724464555 (+1 ms) 2024-11-16T02:34:24,563 INFO [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731724464432 2024-11-16T02:34:24,574 DEBUG [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T02:34:24,574 INFO [RS_OPEN_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T02:34:24,577 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=667222f7cf1b,40547,1731724462323 2024-11-16T02:34:24,579 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 667222f7cf1b,40547,1731724462323, state=OPEN 2024-11-16T02:34:24,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T02:34:24,637 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T02:34:24,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T02:34:24,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T02:34:24,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T02:34:24,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T02:34:24,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T02:34:24,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T02:34:24,639 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=667222f7cf1b,40547,1731724462323 2024-11-16T02:34:24,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T02:34:24,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=667222f7cf1b,40547,1731724462323 in 381 msec 2024-11-16T02:34:24,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T02:34:24,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 907 msec 2024-11-16T02:34:24,657 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T02:34:24,657 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T02:34:24,676 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T02:34:24,677 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=667222f7cf1b,40547,1731724462323, seqNum=-1] 2024-11-16T02:34:24,695 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T02:34:24,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46149, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T02:34:24,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1640 sec 2024-11-16T02:34:24,716 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731724464716, completionTime=-1 2024-11-16T02:34:24,719 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-16T02:34:24,719 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-16T02:34:24,742 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-16T02:34:24,742 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731724524742 2024-11-16T02:34:24,742 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731724584742 2024-11-16T02:34:24,742 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 23 msec 2024-11-16T02:34:24,744 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T02:34:24,750 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=667222f7cf1b,38401,1731724461545-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:24,750 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=667222f7cf1b,38401,1731724461545-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:24,750 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=667222f7cf1b,38401,1731724461545-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:24,752 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-667222f7cf1b:38401, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:24,752 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:24,753 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:24,759 DEBUG [master/667222f7cf1b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T02:34:24,802 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.229sec 2024-11-16T02:34:24,804 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T02:34:24,806 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T02:34:24,807 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T02:34:24,808 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T02:34:24,808 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T02:34:24,809 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=667222f7cf1b,38401,1731724461545-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T02:34:24,810 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=667222f7cf1b,38401,1731724461545-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T02:34:24,819 DEBUG [master/667222f7cf1b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T02:34:24,820 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T02:34:24,821 INFO [master/667222f7cf1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=667222f7cf1b,38401,1731724461545-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:24,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd494aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T02:34:24,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 667222f7cf1b,38401,-1 for getting cluster id 2024-11-16T02:34:24,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T02:34:24,921 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5c137c35-6caf-4e9b-872c-afcb989f7817' 2024-11-16T02:34:24,923 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T02:34:24,924 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5c137c35-6caf-4e9b-872c-afcb989f7817" 2024-11-16T02:34:24,924 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587a2e59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T02:34:24,924 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [667222f7cf1b,38401,-1] 2024-11-16T02:34:24,926 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T02:34:24,928 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T02:34:24,929 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T02:34:24,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75555b5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T02:34:24,932 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T02:34:24,939 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=667222f7cf1b,40547,1731724462323, seqNum=-1] 2024-11-16T02:34:24,939 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T02:34:24,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T02:34:24,960 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=667222f7cf1b,38401,1731724461545 2024-11-16T02:34:24,960 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:43879/hbase 2024-11-16T02:34:24,974 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=363, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=425, ProcessCount=11, AvailableMemoryMB=4573 2024-11-16T02:34:24,993 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:24,997 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:24,998 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:25,003 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72634344, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/hregion-72634344, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:25,018 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72634344/hregion-72634344.1731724465005, exclude list is [], retry=0 2024-11-16T02:34:25,023 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:25,023 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:25,024 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:25,053 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72634344/hregion-72634344.1731724465005 2024-11-16T02:34:25,053 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:25,054 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => fbfd5a3b42ed5d9295fc71cb26171c74, NAME => 'testReplayEditsWrittenIntoWAL,,1731724464994.fbfd5a3b42ed5d9295fc71cb26171c74.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43879/hbase 2024-11-16T02:34:25,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741838_1014 (size=64) 2024-11-16T02:34:25,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741838_1014 (size=64) 2024-11-16T02:34:25,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741838_1014 (size=64) 2024-11-16T02:34:25,068 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731724464994.fbfd5a3b42ed5d9295fc71cb26171c74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:25,071 INFO [StoreOpener-fbfd5a3b42ed5d9295fc71cb26171c74-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,073 INFO [StoreOpener-fbfd5a3b42ed5d9295fc71cb26171c74-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbfd5a3b42ed5d9295fc71cb26171c74 columnFamilyName a 2024-11-16T02:34:25,074 DEBUG [StoreOpener-fbfd5a3b42ed5d9295fc71cb26171c74-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:25,075 INFO [StoreOpener-fbfd5a3b42ed5d9295fc71cb26171c74-1 {}] regionserver.HStore(327): Store=fbfd5a3b42ed5d9295fc71cb26171c74/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:25,075 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,076 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,077 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,078 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,078 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,082 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,086 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:25,087 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fbfd5a3b42ed5d9295fc71cb26171c74; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70577779, jitterRate=0.05169086158275604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T02:34:25,088 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fbfd5a3b42ed5d9295fc71cb26171c74: Writing region info on filesystem at 1731724465068Initializing all the Stores at 1731724465070 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724465070Cleaning up temporary data from old regions at 1731724465078 (+8 ms)Region opened successfully at 1731724465088 (+10 ms) 2024-11-16T02:34:25,088 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fbfd5a3b42ed5d9295fc71cb26171c74, disabling compactions & flushes 2024-11-16T02:34:25,089 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731724464994.fbfd5a3b42ed5d9295fc71cb26171c74. 2024-11-16T02:34:25,089 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731724464994.fbfd5a3b42ed5d9295fc71cb26171c74. 2024-11-16T02:34:25,089 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731724464994.fbfd5a3b42ed5d9295fc71cb26171c74. after waiting 0 ms 2024-11-16T02:34:25,089 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731724464994.fbfd5a3b42ed5d9295fc71cb26171c74. 2024-11-16T02:34:25,089 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731724464994.fbfd5a3b42ed5d9295fc71cb26171c74. 2024-11-16T02:34:25,089 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fbfd5a3b42ed5d9295fc71cb26171c74: Waiting for close lock at 1731724465088Disabling compacts and flushes for region at 1731724465088Disabling writes for close at 1731724465089 (+1 ms)Writing region close event to WAL at 1731724465089Closed at 1731724465089 2024-11-16T02:34:25,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741837_1013 (size=175) 2024-11-16T02:34:25,095 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-72634344/hregion-72634344.1731724465005 not finished, retry = 0 2024-11-16T02:34:25,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741837_1013 (size=175) 2024-11-16T02:34:25,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741837_1013 (size=175) 2024-11-16T02:34:25,206 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:25,206 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72634344:(num 1731724465005) 2024-11-16T02:34:25,208 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-16T02:34:25,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741839_1015 (size=469) 2024-11-16T02:34:25,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741839_1015 (size=469) 2024-11-16T02:34:25,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741839_1015 (size=469) 2024-11-16T02:34:25,249 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-16T02:34:25,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741840_1016 (size=322) 2024-11-16T02:34:25,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741840_1016 (size=322) 2024-11-16T02:34:25,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741840_1016 (size=322) 2024-11-16T02:34:25,307 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-1, size=469 (469bytes) 2024-11-16T02:34:25,308 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-16T02:34:25,308 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-16T02:34:25,308 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-1 2024-11-16T02:34:25,313 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-1 after 3ms 2024-11-16T02:34:25,344 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:25,344 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-1 took 38ms 2024-11-16T02:34:25,349 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-1 so closing down 2024-11-16T02:34:25,349 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:25,353 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-16T02:34:25,386 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000001-wal-1.temp 2024-11-16T02:34:25,387 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:25,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741841_1017 (size=469) 2024-11-16T02:34:25,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741841_1017 (size=469) 2024-11-16T02:34:25,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741841_1017 (size=469) 2024-11-16T02:34:25,400 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-16T02:34:25,403 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000002 2024-11-16T02:34:25,408 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-16T02:34:25,408 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-1, journal: Splitting hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-1, size=469 (469bytes) at 1731724465307Finishing writing output for hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-1 so closing down at 1731724465349 (+42 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000001-wal-1.temp at 1731724465386 (+37 ms)3 split writer threads finished at 1731724465387 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731724465400 (+13 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000002 at 1731724465403 (+3 ms)Processed 2 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731724465408 (+5 ms) 2024-11-16T02:34:25,429 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-2, size=322 (322bytes) 2024-11-16T02:34:25,429 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-2 2024-11-16T02:34:25,430 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-2 after 1ms 2024-11-16T02:34:25,457 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:25,457 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-2 took 28ms 2024-11-16T02:34:25,461 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-2 so closing down 2024-11-16T02:34:25,461 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:25,464 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-16T02:34:25,490 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000002-wal-2.temp 2024-11-16T02:34:25,490 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:25,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741842_1018 (size=322) 2024-11-16T02:34:25,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741842_1018 (size=322) 2024-11-16T02:34:25,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741842_1018 (size=322) 2024-11-16T02:34:25,501 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T02:34:25,528 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:25,531 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-16T02:34:25,533 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-16T02:34:25,533 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-2, journal: Splitting hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-2, size=322 (322bytes) at 1731724465429Finishing writing output for hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-2 so closing down at 1731724465461 (+32 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000002-wal-2.temp at 1731724465490 (+29 ms)3 split writer threads finished at 1731724465490Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731724465501 (+11 ms)Processed 1 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731724465533 (+32 ms) 2024-11-16T02:34:25,533 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:25,536 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:25,548 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal.1731724465537, exclude list is [], retry=0 2024-11-16T02:34:25,551 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:25,552 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:25,552 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:25,577 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal.1731724465537 2024-11-16T02:34:25,577 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:41131:41131)] 2024-11-16T02:34:25,578 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => fbfd5a3b42ed5d9295fc71cb26171c74, NAME => 'testReplayEditsWrittenIntoWAL,,1731724464994.fbfd5a3b42ed5d9295fc71cb26171c74.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:25,578 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731724464994.fbfd5a3b42ed5d9295fc71cb26171c74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:25,578 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,578 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,580 INFO [StoreOpener-fbfd5a3b42ed5d9295fc71cb26171c74-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,582 INFO [StoreOpener-fbfd5a3b42ed5d9295fc71cb26171c74-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbfd5a3b42ed5d9295fc71cb26171c74 columnFamilyName a 2024-11-16T02:34:25,582 DEBUG [StoreOpener-fbfd5a3b42ed5d9295fc71cb26171c74-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:25,583 INFO [StoreOpener-fbfd5a3b42ed5d9295fc71cb26171c74-1 {}] regionserver.HStore(327): Store=fbfd5a3b42ed5d9295fc71cb26171c74/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:25,584 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,585 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,588 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,590 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000002 2024-11-16T02:34:25,619 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:25,626 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000002 2024-11-16T02:34:25,630 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fbfd5a3b42ed5d9295fc71cb26171c74 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-16T02:34:25,681 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/.tmp/a/015bb2665690437fbd2dcc7d8517acc5 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731724465206/Put/seqid=0 2024-11-16T02:34:25,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741844_1020 (size=5170) 2024-11-16T02:34:25,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741844_1020 (size=5170) 2024-11-16T02:34:25,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741844_1020 (size=5170) 2024-11-16T02:34:25,697 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/.tmp/a/015bb2665690437fbd2dcc7d8517acc5 2024-11-16T02:34:25,739 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/.tmp/a/015bb2665690437fbd2dcc7d8517acc5 as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/a/015bb2665690437fbd2dcc7d8517acc5 2024-11-16T02:34:25,749 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/a/015bb2665690437fbd2dcc7d8517acc5, entries=2, sequenceid=2, filesize=5.0 K 2024-11-16T02:34:25,753 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for fbfd5a3b42ed5d9295fc71cb26171c74 in 124ms, sequenceid=2, compaction requested=false; wal=null 2024-11-16T02:34:25,755 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/0000000000000000002 2024-11-16T02:34:25,756 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,756 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,759 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fbfd5a3b42ed5d9295fc71cb26171c74 2024-11-16T02:34:25,762 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/fbfd5a3b42ed5d9295fc71cb26171c74/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-16T02:34:25,764 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fbfd5a3b42ed5d9295fc71cb26171c74; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64141055, jitterRate=-0.04422380030155182}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T02:34:25,764 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fbfd5a3b42ed5d9295fc71cb26171c74: Writing region info on filesystem at 1731724465578Initializing all the Stores at 1731724465580 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724465580Obtaining lock to block concurrent updates at 1731724465630 (+50 ms)Preparing flush snapshotting stores in fbfd5a3b42ed5d9295fc71cb26171c74 at 1731724465630Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731724464994.fbfd5a3b42ed5d9295fc71cb26171c74., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731724465633 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731724464994.fbfd5a3b42ed5d9295fc71cb26171c74. at 1731724465634 (+1 ms)Flushing fbfd5a3b42ed5d9295fc71cb26171c74/a: creating writer at 1731724465635 (+1 ms)Flushing fbfd5a3b42ed5d9295fc71cb26171c74/a: appending metadata at 1731724465669 (+34 ms)Flushing fbfd5a3b42ed5d9295fc71cb26171c74/a: closing flushed file at 1731724465671 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4722f1b6: reopening flushed file at 1731724465737 (+66 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for fbfd5a3b42ed5d9295fc71cb26171c74 in 124ms, sequenceid=2, compaction requested=false; wal=null at 1731724465754 (+17 ms)Cleaning up temporary data from old regions at 1731724465756 (+2 ms)Region opened successfully at 1731724465764 (+8 ms) 2024-11-16T02:34:25,791 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=375 (was 363) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:52628 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:50326 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:43879/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:50148 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:52504 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:35610 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:35632 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=425 (was 425), ProcessCount=11 (was 11), AvailableMemoryMB=4547 (was 4573) 2024-11-16T02:34:25,801 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=375, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=425, ProcessCount=11, AvailableMemoryMB=4546 2024-11-16T02:34:25,816 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:25,818 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:25,819 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:25,823 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-91881788, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/hregion-91881788, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:25,835 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-91881788/hregion-91881788.1731724465823, exclude list is [], retry=0 2024-11-16T02:34:25,839 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:25,839 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:25,839 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:25,864 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-91881788/hregion-91881788.1731724465823 2024-11-16T02:34:25,864 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131)] 2024-11-16T02:34:25,865 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 03113d4d6f0f4b597db9a215683409de, NAME => 'testReplayEditsWrittenIntoWAL,,1731724465817.03113d4d6f0f4b597db9a215683409de.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43879/hbase 2024-11-16T02:34:25,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741846_1022 (size=64) 2024-11-16T02:34:25,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741846_1022 (size=64) 2024-11-16T02:34:25,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741846_1022 (size=64) 2024-11-16T02:34:25,878 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731724465817.03113d4d6f0f4b597db9a215683409de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:25,880 INFO [StoreOpener-03113d4d6f0f4b597db9a215683409de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:25,883 INFO [StoreOpener-03113d4d6f0f4b597db9a215683409de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03113d4d6f0f4b597db9a215683409de columnFamilyName a 2024-11-16T02:34:25,883 DEBUG [StoreOpener-03113d4d6f0f4b597db9a215683409de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:25,884 INFO [StoreOpener-03113d4d6f0f4b597db9a215683409de-1 {}] regionserver.HStore(327): Store=03113d4d6f0f4b597db9a215683409de/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:25,884 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:25,885 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:25,886 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:25,886 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:25,886 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:25,888 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:25,892 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:25,893 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 03113d4d6f0f4b597db9a215683409de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62007673, jitterRate=-0.07601366937160492}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T02:34:25,893 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 03113d4d6f0f4b597db9a215683409de: Writing region info on filesystem at 1731724465878Initializing all the Stores at 1731724465880 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724465880Cleaning up temporary data from old regions at 1731724465886 (+6 ms)Region opened successfully at 1731724465893 (+7 ms) 2024-11-16T02:34:25,893 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 03113d4d6f0f4b597db9a215683409de, disabling compactions & flushes 2024-11-16T02:34:25,893 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731724465817.03113d4d6f0f4b597db9a215683409de. 2024-11-16T02:34:25,893 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731724465817.03113d4d6f0f4b597db9a215683409de. 2024-11-16T02:34:25,894 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731724465817.03113d4d6f0f4b597db9a215683409de. after waiting 0 ms 2024-11-16T02:34:25,894 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731724465817.03113d4d6f0f4b597db9a215683409de. 2024-11-16T02:34:25,894 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731724465817.03113d4d6f0f4b597db9a215683409de. 2024-11-16T02:34:25,894 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 03113d4d6f0f4b597db9a215683409de: Waiting for close lock at 1731724465893Disabling compacts and flushes for region at 1731724465893Disabling writes for close at 1731724465894 (+1 ms)Writing region close event to WAL at 1731724465894Closed at 1731724465894 2024-11-16T02:34:25,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741845_1021 (size=175) 2024-11-16T02:34:25,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741845_1021 (size=175) 2024-11-16T02:34:25,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741845_1021 (size=175) 2024-11-16T02:34:25,901 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:25,901 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-91881788:(num 1731724465823) 2024-11-16T02:34:25,902 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-16T02:34:25,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741847_1023 (size=469) 2024-11-16T02:34:25,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741847_1023 (size=469) 2024-11-16T02:34:25,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741847_1023 (size=469) 2024-11-16T02:34:25,940 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-16T02:34:25,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741848_1024 (size=322) 2024-11-16T02:34:25,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741848_1024 (size=322) 2024-11-16T02:34:25,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741848_1024 (size=322) 2024-11-16T02:34:25,990 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-2, size=322 (322bytes) 2024-11-16T02:34:25,990 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-2 2024-11-16T02:34:25,991 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-2 after 1ms 2024-11-16T02:34:26,017 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:26,017 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-2 took 27ms 2024-11-16T02:34:26,020 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-2 so closing down 2024-11-16T02:34:26,020 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:26,022 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-16T02:34:26,047 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002-wal-2.temp 2024-11-16T02:34:26,047 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:26,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741849_1025 (size=322) 2024-11-16T02:34:26,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741849_1025 (size=322) 2024-11-16T02:34:26,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741849_1025 (size=322) 2024-11-16T02:34:26,057 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T02:34:26,059 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002 2024-11-16T02:34:26,059 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-16T02:34:26,059 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-2, journal: Splitting hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-2, size=322 (322bytes) at 1731724465990Finishing writing output for hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-2 so closing down at 1731724466020 (+30 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002-wal-2.temp at 1731724466047 (+27 ms)3 split writer threads finished at 1731724466047Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731724466057 (+10 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002 at 1731724466059 (+2 ms)Processed 1 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731724466059 2024-11-16T02:34:26,074 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-1, size=469 (469bytes) 2024-11-16T02:34:26,074 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-1 2024-11-16T02:34:26,075 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-1 after 1ms 2024-11-16T02:34:26,101 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:26,101 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-1 took 27ms 2024-11-16T02:34:26,104 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-1 so closing down 2024-11-16T02:34:26,104 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:26,107 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-16T02:34:26,132 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000001-wal-1.temp 2024-11-16T02:34:26,132 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:26,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741850_1026 (size=469) 2024-11-16T02:34:26,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741850_1026 (size=469) 2024-11-16T02:34:26,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741850_1026 (size=469) 2024-11-16T02:34:26,140 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-16T02:34:26,166 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:26,169 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002, length=322 2024-11-16T02:34:26,171 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002 2024-11-16T02:34:26,171 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-16T02:34:26,171 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-1, journal: Splitting hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-1, size=469 (469bytes) at 1731724466074Finishing writing output for hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-1 so closing down at 1731724466104 (+30 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000001-wal-1.temp at 1731724466132 (+28 ms)3 split writer threads finished at 1731724466132Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731724466140 (+8 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002 at 1731724466171 (+31 ms)Processed 2 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731724466171 2024-11-16T02:34:26,171 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:26,173 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:26,187 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal.1731724466174, exclude list is [], retry=0 2024-11-16T02:34:26,190 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:26,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:26,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:26,217 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal.1731724466174 2024-11-16T02:34:26,217 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:26,217 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 03113d4d6f0f4b597db9a215683409de, NAME => 'testReplayEditsWrittenIntoWAL,,1731724465817.03113d4d6f0f4b597db9a215683409de.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:26,217 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731724465817.03113d4d6f0f4b597db9a215683409de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:26,218 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:26,218 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:26,220 INFO [StoreOpener-03113d4d6f0f4b597db9a215683409de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:26,222 INFO [StoreOpener-03113d4d6f0f4b597db9a215683409de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03113d4d6f0f4b597db9a215683409de columnFamilyName a 2024-11-16T02:34:26,223 DEBUG [StoreOpener-03113d4d6f0f4b597db9a215683409de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:26,224 INFO [StoreOpener-03113d4d6f0f4b597db9a215683409de-1 {}] regionserver.HStore(327): Store=03113d4d6f0f4b597db9a215683409de/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:26,224 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:26,225 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:26,228 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:26,229 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002 2024-11-16T02:34:26,256 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:26,258 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002 2024-11-16T02:34:26,258 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 03113d4d6f0f4b597db9a215683409de 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-16T02:34:26,274 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/.tmp/a/ac85da9d958044789c0a7c12bf5aa9bd is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731724465902/Put/seqid=0 2024-11-16T02:34:26,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741852_1028 (size=5170) 2024-11-16T02:34:26,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741852_1028 (size=5170) 2024-11-16T02:34:26,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741852_1028 (size=5170) 2024-11-16T02:34:26,283 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/.tmp/a/ac85da9d958044789c0a7c12bf5aa9bd 2024-11-16T02:34:26,292 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/.tmp/a/ac85da9d958044789c0a7c12bf5aa9bd as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/a/ac85da9d958044789c0a7c12bf5aa9bd 2024-11-16T02:34:26,301 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/a/ac85da9d958044789c0a7c12bf5aa9bd, entries=2, sequenceid=2, filesize=5.0 K 2024-11-16T02:34:26,302 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 03113d4d6f0f4b597db9a215683409de in 44ms, sequenceid=2, compaction requested=false; wal=null 2024-11-16T02:34:26,303 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/0000000000000000002 2024-11-16T02:34:26,303 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:26,303 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:26,306 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 03113d4d6f0f4b597db9a215683409de 2024-11-16T02:34:26,309 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/03113d4d6f0f4b597db9a215683409de/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-16T02:34:26,311 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 03113d4d6f0f4b597db9a215683409de; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66228337, jitterRate=-0.013120874762535095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T02:34:26,311 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 03113d4d6f0f4b597db9a215683409de: Writing region info on filesystem at 1731724466218Initializing all the Stores at 1731724466219 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724466219Obtaining lock to block concurrent updates at 1731724466258 (+39 ms)Preparing flush snapshotting stores in 03113d4d6f0f4b597db9a215683409de at 1731724466258Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731724465817.03113d4d6f0f4b597db9a215683409de., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731724466258Flushing stores of testReplayEditsWrittenIntoWAL,,1731724465817.03113d4d6f0f4b597db9a215683409de. at 1731724466258Flushing 03113d4d6f0f4b597db9a215683409de/a: creating writer at 1731724466258Flushing 03113d4d6f0f4b597db9a215683409de/a: appending metadata at 1731724466273 (+15 ms)Flushing 03113d4d6f0f4b597db9a215683409de/a: closing flushed file at 1731724466273Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@249a9509: reopening flushed file at 1731724466291 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 03113d4d6f0f4b597db9a215683409de in 44ms, sequenceid=2, compaction requested=false; wal=null at 1731724466302 (+11 ms)Cleaning up temporary data from old regions at 1731724466303 (+1 ms)Region opened successfully at 1731724466311 (+8 ms) 2024-11-16T02:34:26,326 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=385 (was 375) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:50370 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:52694 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:35610 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:35700 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:50148 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:52504 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=425 (was 425), ProcessCount=11 (was 11), AvailableMemoryMB=4528 (was 4546) 2024-11-16T02:34:26,337 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=425, ProcessCount=11, AvailableMemoryMB=4528 2024-11-16T02:34:26,351 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:26,353 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:26,354 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:26,358 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-48290699, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/hregion-48290699, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:26,370 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-48290699/hregion-48290699.1731724466358, exclude list is [], retry=0 2024-11-16T02:34:26,374 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:26,374 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:26,374 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:26,400 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-48290699/hregion-48290699.1731724466358 2024-11-16T02:34:26,400 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:38477:38477)] 2024-11-16T02:34:26,401 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ca3ffdd744a20c6b5afedb7cf5773ad8, NAME => 'testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43879/hbase 2024-11-16T02:34:26,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741854_1030 (size=64) 2024-11-16T02:34:26,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741854_1030 (size=64) 2024-11-16T02:34:26,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741854_1030 (size=64) 2024-11-16T02:34:26,414 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:26,416 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:26,417 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca3ffdd744a20c6b5afedb7cf5773ad8 columnFamilyName a 2024-11-16T02:34:26,417 DEBUG [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:26,418 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] regionserver.HStore(327): Store=ca3ffdd744a20c6b5afedb7cf5773ad8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:26,418 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:26,420 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca3ffdd744a20c6b5afedb7cf5773ad8 columnFamilyName b 2024-11-16T02:34:26,420 DEBUG [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:26,420 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] regionserver.HStore(327): Store=ca3ffdd744a20c6b5afedb7cf5773ad8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:26,420 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:26,422 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca3ffdd744a20c6b5afedb7cf5773ad8 columnFamilyName c 2024-11-16T02:34:26,422 DEBUG [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:26,423 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] regionserver.HStore(327): Store=ca3ffdd744a20c6b5afedb7cf5773ad8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:26,423 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:26,424 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:26,424 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:26,425 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:26,425 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:26,426 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:26,427 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:26,430 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:26,430 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca3ffdd744a20c6b5afedb7cf5773ad8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73855884, jitterRate=0.10053843259811401}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:26,431 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca3ffdd744a20c6b5afedb7cf5773ad8: Writing region info on filesystem at 1731724466414Initializing all the Stores at 1731724466415 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724466415Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724466415Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724466415Cleaning up temporary data from old regions at 1731724466425 (+10 ms)Region opened successfully at 1731724466431 (+6 ms) 2024-11-16T02:34:26,431 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ca3ffdd744a20c6b5afedb7cf5773ad8, disabling compactions & flushes 2024-11-16T02:34:26,431 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8. 2024-11-16T02:34:26,431 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8. 2024-11-16T02:34:26,431 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8. after waiting 0 ms 2024-11-16T02:34:26,431 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8. 2024-11-16T02:34:26,432 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8. 2024-11-16T02:34:26,432 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ca3ffdd744a20c6b5afedb7cf5773ad8: Waiting for close lock at 1731724466431Disabling compacts and flushes for region at 1731724466431Disabling writes for close at 1731724466431Writing region close event to WAL at 1731724466432 (+1 ms)Closed at 1731724466432 2024-11-16T02:34:26,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741853_1029 (size=175) 2024-11-16T02:34:26,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741853_1029 (size=175) 2024-11-16T02:34:26,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741853_1029 (size=175) 2024-11-16T02:34:26,438 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:26,439 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-48290699:(num 1731724466358) 2024-11-16T02:34:26,439 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:26,441 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:26,455 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442, exclude list is [], retry=0 2024-11-16T02:34:26,459 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:26,459 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:26,460 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:26,486 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442 2024-11-16T02:34:26,486 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:26,692 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442, size=0 (0bytes) 2024-11-16T02:34:26,692 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442 might be still open, length is 0 2024-11-16T02:34:26,692 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442 2024-11-16T02:34:26,693 WARN [IPC Server handler 3 on default port 43879 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-16T02:34:26,694 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442 after 1ms 2024-11-16T02:34:29,661 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:35720 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33993:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35720 dst: /127.0.0.1:33993 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33993 remote=/127.0.0.1:35720]. Total timeout mills is 60000, 57001 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:29,663 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:52728 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39531:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52728 dst: /127.0.0.1:39531 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:29,663 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:50394 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50394 dst: /127.0.0.1:40973 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:29,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741855_1032 (size=497775) 2024-11-16T02:34:29,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741855_1032 (size=497775) 2024-11-16T02:34:29,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741855_1032 (size=497775) 2024-11-16T02:34:30,042 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T02:34:30,097 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T02:34:30,694 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442 after 4002ms 2024-11-16T02:34:30,726 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:30,727 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442 took 4035ms 2024-11-16T02:34:30,733 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731724466442.temp 2024-11-16T02:34:30,761 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/0000000000000000001-wal.1731724466442.temp 2024-11-16T02:34:30,824 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442; continuing. 2024-11-16T02:34:30,825 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442 so closing down 2024-11-16T02:34:30,825 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:30,825 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:30,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741856_1033 (size=497783) 2024-11-16T02:34:30,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741856_1033 (size=497783) 2024-11-16T02:34:30,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741856_1033 (size=497783) 2024-11-16T02:34:30,829 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/0000000000000000001-wal.1731724466442.temp (wrote 3002 edits, skipped 0 edits in 43 ms) 2024-11-16T02:34:30,831 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/0000000000000000001-wal.1731724466442.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/0000000000000003002 2024-11-16T02:34:30,831 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 103 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442, size=0, length=0, corrupted=false, cancelled=false 2024-11-16T02:34:30,832 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442, journal: Splitting hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442, size=0 (0bytes) at 1731724466692Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/0000000000000000001-wal.1731724466442.temp at 1731724470761 (+4069 ms)Split 1024 edits, skipped 0 edits. at 1731724470791 (+30 ms)Split 2048 edits, skipped 0 edits. at 1731724470808 (+17 ms)Finishing writing output for hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442 so closing down at 1731724470825 (+17 ms)3 split writer threads finished at 1731724470825Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/0000000000000000001-wal.1731724466442.temp (wrote 3002 edits, skipped 0 edits in 43 ms) at 1731724470829 (+4 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/0000000000000000001-wal.1731724466442.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/0000000000000003002 at 1731724470831 (+2 ms)Processed 3002 edits across 1 Regions in 103 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442, size=0, length=0, corrupted=false, cancelled=false at 1731724470831 2024-11-16T02:34:30,834 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442 to hdfs://localhost:43879/hbase/oldWALs/wal.1731724466442 2024-11-16T02:34:30,836 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/0000000000000003002 2024-11-16T02:34:30,836 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:30,838 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:30,853 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724470839, exclude list is [], retry=0 2024-11-16T02:34:30,856 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:30,857 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:30,857 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:30,886 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724470839 2024-11-16T02:34:30,887 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477)] 2024-11-16T02:34:30,887 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:30,889 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:30,890 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca3ffdd744a20c6b5afedb7cf5773ad8 columnFamilyName a 2024-11-16T02:34:30,890 DEBUG [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:30,891 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] regionserver.HStore(327): Store=ca3ffdd744a20c6b5afedb7cf5773ad8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:30,891 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:30,892 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca3ffdd744a20c6b5afedb7cf5773ad8 columnFamilyName b 2024-11-16T02:34:30,892 DEBUG [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:30,893 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] regionserver.HStore(327): Store=ca3ffdd744a20c6b5afedb7cf5773ad8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:30,893 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:30,894 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca3ffdd744a20c6b5afedb7cf5773ad8 columnFamilyName c 2024-11-16T02:34:30,894 DEBUG [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:30,894 INFO [StoreOpener-ca3ffdd744a20c6b5afedb7cf5773ad8-1 {}] regionserver.HStore(327): Store=ca3ffdd744a20c6b5afedb7cf5773ad8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:30,895 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:30,895 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:30,897 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:30,898 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/0000000000000003002 2024-11-16T02:34:30,925 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:30,959 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-16T02:34:31,286 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ca3ffdd744a20c6b5afedb7cf5773ad8 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-16T02:34:31,325 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/a/e48a2736ff904ba69bf6996b7f72d1b8 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731724466493/Put/seqid=0 2024-11-16T02:34:31,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741858_1035 (size=50463) 2024-11-16T02:34:31,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741858_1035 (size=50463) 2024-11-16T02:34:31,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741858_1035 (size=50463) 2024-11-16T02:34:31,334 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/a/e48a2736ff904ba69bf6996b7f72d1b8 2024-11-16T02:34:31,341 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/a/e48a2736ff904ba69bf6996b7f72d1b8 as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/a/e48a2736ff904ba69bf6996b7f72d1b8 2024-11-16T02:34:31,348 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/a/e48a2736ff904ba69bf6996b7f72d1b8, entries=754, sequenceid=754, filesize=49.3 K 2024-11-16T02:34:31,349 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for ca3ffdd744a20c6b5afedb7cf5773ad8 in 64ms, sequenceid=754, compaction requested=false; wal=null 2024-11-16T02:34:31,365 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-16T02:34:31,365 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ca3ffdd744a20c6b5afedb7cf5773ad8 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-16T02:34:31,373 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/a/c4459a5163964f148d573da409adcee2 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731724466532/Put/seqid=0 2024-11-16T02:34:31,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741859_1036 (size=20072) 2024-11-16T02:34:31,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741859_1036 (size=20072) 2024-11-16T02:34:31,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741859_1036 (size=20072) 2024-11-16T02:34:31,382 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/a/c4459a5163964f148d573da409adcee2 2024-11-16T02:34:31,407 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/b/791d17739e114e549245512b7ccf8ea5 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731724466561/Put/seqid=0 2024-11-16T02:34:31,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741860_1037 (size=35835) 2024-11-16T02:34:31,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741860_1037 (size=35835) 2024-11-16T02:34:31,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741860_1037 (size=35835) 2024-11-16T02:34:31,417 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/b/791d17739e114e549245512b7ccf8ea5 2024-11-16T02:34:31,423 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/a/c4459a5163964f148d573da409adcee2 as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/a/c4459a5163964f148d573da409adcee2 2024-11-16T02:34:31,429 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/a/c4459a5163964f148d573da409adcee2, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-16T02:34:31,431 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/b/791d17739e114e549245512b7ccf8ea5 as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/b/791d17739e114e549245512b7ccf8ea5 2024-11-16T02:34:31,437 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/b/791d17739e114e549245512b7ccf8ea5, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-16T02:34:31,437 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ca3ffdd744a20c6b5afedb7cf5773ad8 in 72ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-16T02:34:31,460 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-16T02:34:31,460 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ca3ffdd744a20c6b5afedb7cf5773ad8 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-16T02:34:31,471 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/b/fc48fe577b1b431eb4e15f10ab93cd00 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731724466578/Put/seqid=0 2024-11-16T02:34:31,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741861_1038 (size=35082) 2024-11-16T02:34:31,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741861_1038 (size=35082) 2024-11-16T02:34:31,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741861_1038 (size=35082) 2024-11-16T02:34:31,483 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/b/fc48fe577b1b431eb4e15f10ab93cd00 2024-11-16T02:34:31,509 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/c/cdbdc9e92a214568aced99f26eea9e28 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731724466605/Put/seqid=0 2024-11-16T02:34:31,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741862_1039 (size=20825) 2024-11-16T02:34:31,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741862_1039 (size=20825) 2024-11-16T02:34:31,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741862_1039 (size=20825) 2024-11-16T02:34:31,517 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/c/cdbdc9e92a214568aced99f26eea9e28 2024-11-16T02:34:31,525 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/b/fc48fe577b1b431eb4e15f10ab93cd00 as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/b/fc48fe577b1b431eb4e15f10ab93cd00 2024-11-16T02:34:31,532 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/b/fc48fe577b1b431eb4e15f10ab93cd00, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-16T02:34:31,534 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/c/cdbdc9e92a214568aced99f26eea9e28 as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/c/cdbdc9e92a214568aced99f26eea9e28 2024-11-16T02:34:31,541 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/c/cdbdc9e92a214568aced99f26eea9e28, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-16T02:34:31,541 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ca3ffdd744a20c6b5afedb7cf5773ad8 in 81ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-16T02:34:31,559 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731724466661/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8. 2024-11-16T02:34:31,562 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/0000000000000003002 2024-11-16T02:34:31,563 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-16T02:34:31,563 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ca3ffdd744a20c6b5afedb7cf5773ad8 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-16T02:34:31,571 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/c/c7674150dcf9461cafae8aa5a1e4f954 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731724466615/Put/seqid=0 2024-11-16T02:34:31,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741863_1040 (size=50301) 2024-11-16T02:34:31,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741863_1040 (size=50301) 2024-11-16T02:34:31,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741863_1040 (size=50301) 2024-11-16T02:34:31,579 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/c/c7674150dcf9461cafae8aa5a1e4f954 2024-11-16T02:34:31,586 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7674150dcf9461cafae8aa5a1e4f954 2024-11-16T02:34:31,588 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/.tmp/c/c7674150dcf9461cafae8aa5a1e4f954 as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/c/c7674150dcf9461cafae8aa5a1e4f954 2024-11-16T02:34:31,595 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7674150dcf9461cafae8aa5a1e4f954 2024-11-16T02:34:31,596 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/c/c7674150dcf9461cafae8aa5a1e4f954, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-16T02:34:31,596 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for ca3ffdd744a20c6b5afedb7cf5773ad8 in 33ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-16T02:34:31,597 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/0000000000000003002 2024-11-16T02:34:31,599 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:31,599 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:31,600 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T02:34:31,602 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca3ffdd744a20c6b5afedb7cf5773ad8 2024-11-16T02:34:31,605 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenIntoWAL/ca3ffdd744a20c6b5afedb7cf5773ad8/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-16T02:34:31,606 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca3ffdd744a20c6b5afedb7cf5773ad8; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66674248, jitterRate=-0.006476283073425293}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T02:34:31,606 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca3ffdd744a20c6b5afedb7cf5773ad8: Writing region info on filesystem at 1731724470887Initializing all the Stores at 1731724470888 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724470888Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724470889 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724470889Cleaning up temporary data from old regions at 1731724471599 (+710 ms)Region opened successfully at 1731724471606 (+7 ms) 2024-11-16T02:34:31,674 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ca3ffdd744a20c6b5afedb7cf5773ad8, disabling compactions & flushes 2024-11-16T02:34:31,674 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8. 2024-11-16T02:34:31,675 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8. 2024-11-16T02:34:31,675 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8. after waiting 0 ms 2024-11-16T02:34:31,675 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8. 2024-11-16T02:34:31,677 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731724466352.ca3ffdd744a20c6b5afedb7cf5773ad8. 2024-11-16T02:34:31,677 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ca3ffdd744a20c6b5afedb7cf5773ad8: Waiting for close lock at 1731724471674Disabling compacts and flushes for region at 1731724471674Disabling writes for close at 1731724471675 (+1 ms)Writing region close event to WAL at 1731724471677 (+2 ms)Closed at 1731724471677 2024-11-16T02:34:31,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741857_1034 (size=175) 2024-11-16T02:34:31,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741857_1034 (size=175) 2024-11-16T02:34:31,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741857_1034 (size=175) 2024-11-16T02:34:31,684 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:31,684 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731724470839) 2024-11-16T02:34:31,700 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=399 (was 385) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:43879 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:46059 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33733 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1297241481_22 at /127.0.0.1:35756 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1297241481_22 at /127.0.0.1:52746 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1297241481_22 at /127.0.0.1:50444 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1297241481_22 at /127.0.0.1:35790 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33733 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46059 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:43879 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=391 (was 425), ProcessCount=11 (was 11), AvailableMemoryMB=4448 (was 4528) 2024-11-16T02:34:31,712 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=399, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=391, ProcessCount=11, AvailableMemoryMB=4444 2024-11-16T02:34:31,728 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:31,731 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:31,732 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:31,735 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-46655072, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/hregion-46655072, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:31,748 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-46655072/hregion-46655072.1731724471736, exclude list is [], retry=0 2024-11-16T02:34:31,752 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:31,752 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:31,752 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:31,778 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-46655072/hregion-46655072.1731724471736 2024-11-16T02:34:31,779 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:31,779 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 924a20e1b2534c0e45f0d44c6d028459, NAME => 'test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43879/hbase 2024-11-16T02:34:31,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741865_1042 (size=43) 2024-11-16T02:34:31,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741865_1042 (size=43) 2024-11-16T02:34:31,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741865_1042 (size=43) 2024-11-16T02:34:31,790 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:31,791 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:31,793 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 924a20e1b2534c0e45f0d44c6d028459 columnFamilyName a 2024-11-16T02:34:31,793 DEBUG [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:31,794 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] regionserver.HStore(327): Store=924a20e1b2534c0e45f0d44c6d028459/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:31,794 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:31,795 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 924a20e1b2534c0e45f0d44c6d028459 columnFamilyName b 2024-11-16T02:34:31,795 DEBUG [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:31,796 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] regionserver.HStore(327): Store=924a20e1b2534c0e45f0d44c6d028459/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:31,796 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:31,798 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 924a20e1b2534c0e45f0d44c6d028459 columnFamilyName c 2024-11-16T02:34:31,798 DEBUG [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:31,798 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] regionserver.HStore(327): Store=924a20e1b2534c0e45f0d44c6d028459/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:31,798 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:31,799 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:31,800 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:31,801 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:31,801 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:31,801 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:31,803 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:31,805 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:31,806 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 924a20e1b2534c0e45f0d44c6d028459; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62266142, jitterRate=-0.07216218113899231}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:31,807 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 924a20e1b2534c0e45f0d44c6d028459: Writing region info on filesystem at 1731724471790Initializing all the Stores at 1731724471791 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724471791Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724471791Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724471791Cleaning up temporary data from old regions at 1731724471801 (+10 ms)Region opened successfully at 1731724471807 (+6 ms) 2024-11-16T02:34:31,807 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 924a20e1b2534c0e45f0d44c6d028459, disabling compactions & flushes 2024-11-16T02:34:31,807 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459. 2024-11-16T02:34:31,807 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459. 2024-11-16T02:34:31,807 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459. after waiting 0 ms 2024-11-16T02:34:31,807 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459. 2024-11-16T02:34:31,808 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459. 2024-11-16T02:34:31,808 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 924a20e1b2534c0e45f0d44c6d028459: Waiting for close lock at 1731724471807Disabling compacts and flushes for region at 1731724471807Disabling writes for close at 1731724471807Writing region close event to WAL at 1731724471808 (+1 ms)Closed at 1731724471808 2024-11-16T02:34:31,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741864_1041 (size=175) 2024-11-16T02:34:31,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741864_1041 (size=175) 2024-11-16T02:34:31,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741864_1041 (size=175) 2024-11-16T02:34:31,814 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:31,814 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-46655072:(num 1731724471736) 2024-11-16T02:34:31,814 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:31,816 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:31,835 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817, exclude list is [], retry=0 2024-11-16T02:34:31,838 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:31,839 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:31,839 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:31,865 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817 2024-11-16T02:34:31,865 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477)] 2024-11-16T02:34:31,930 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T02:34:31,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T02:34:31,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T02:34:31,933 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T02:34:31,934 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T02:34:31,934 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T02:34:31,935 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-16T02:34:31,935 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-16T02:34:32,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741866_1043 (size=371388) 2024-11-16T02:34:32,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741866_1043 (size=371388) 2024-11-16T02:34:32,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741866_1043 (size=371388) 2024-11-16T02:34:32,031 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817, size=362.7 K (371388bytes) 2024-11-16T02:34:32,031 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817 2024-11-16T02:34:32,032 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817 after 0ms 2024-11-16T02:34:32,061 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:32,063 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817 took 32ms 2024-11-16T02:34:32,067 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731724471817.temp 2024-11-16T02:34:32,109 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000000001-wal.1731724471817.temp 2024-11-16T02:34:32,113 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817 so closing down 2024-11-16T02:34:32,113 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:32,131 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:32,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741867_1044 (size=371388) 2024-11-16T02:34:32,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741867_1044 (size=371388) 2024-11-16T02:34:32,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741867_1044 (size=371388) 2024-11-16T02:34:32,135 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000000001-wal.1731724471817.temp (wrote 3000 edits, skipped 0 edits in 21 ms) 2024-11-16T02:34:32,137 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000000001-wal.1731724471817.temp to hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000003000 2024-11-16T02:34:32,137 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-16T02:34:32,137 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817, journal: Splitting hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817, size=362.7 K (371388bytes) at 1731724472031Split 1024 edits, skipped 0 edits. at 1731724472082 (+51 ms)Split 2048 edits, skipped 0 edits. at 1731724472098 (+16 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000000001-wal.1731724471817.temp at 1731724472110 (+12 ms)Finishing writing output for hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817 so closing down at 1731724472113 (+3 ms)3 split writer threads finished at 1731724472131 (+18 ms)Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000000001-wal.1731724471817.temp (wrote 3000 edits, skipped 0 edits in 21 ms) at 1731724472135 (+4 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000000001-wal.1731724471817.temp to hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000003000 at 1731724472137 (+2 ms)Processed 3000 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1731724472137 2024-11-16T02:34:32,140 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724471817 to hdfs://localhost:43879/hbase/oldWALs/wal.1731724471817 2024-11-16T02:34:32,141 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000003000 2024-11-16T02:34:32,141 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:32,143 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:32,157 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144, exclude list is [], retry=0 2024-11-16T02:34:32,161 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:32,161 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:32,162 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:32,197 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144 2024-11-16T02:34:32,198 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477)] 2024-11-16T02:34:32,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741868_1045 (size=371515) 2024-11-16T02:34:32,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741868_1045 (size=371515) 2024-11-16T02:34:32,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741868_1045 (size=371515) 2024-11-16T02:34:32,366 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144, size=362.8 K (371515bytes) 2024-11-16T02:34:32,366 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144 2024-11-16T02:34:32,366 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144 after 0ms 2024-11-16T02:34:32,398 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:32,400 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144 took 35ms 2024-11-16T02:34:32,405 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731724472144.temp 2024-11-16T02:34:32,461 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144 so closing down 2024-11-16T02:34:32,461 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:32,476 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000003001-wal.1731724472144.temp 2024-11-16T02:34:32,487 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:32,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741869_1046 (size=371515) 2024-11-16T02:34:32,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741869_1046 (size=371515) 2024-11-16T02:34:32,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741869_1046 (size=371515) 2024-11-16T02:34:32,500 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000003001-wal.1731724472144.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-11-16T02:34:32,502 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000003001-wal.1731724472144.temp to hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000006000 2024-11-16T02:34:32,502 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-16T02:34:32,502 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144, journal: Splitting hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144, size=362.8 K (371515bytes) at 1731724472366Split 1024 edits, skipped 0 edits. at 1731724472415 (+49 ms)Split 2048 edits, skipped 0 edits. at 1731724472453 (+38 ms)Finishing writing output for hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144 so closing down at 1731724472461 (+8 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000003001-wal.1731724472144.temp at 1731724472476 (+15 ms)3 split writer threads finished at 1731724472487 (+11 ms)Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000003001-wal.1731724472144.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1731724472500 (+13 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000003001-wal.1731724472144.temp to hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000006000 at 1731724472502 (+2 ms)Processed 3000 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1731724472502 2024-11-16T02:34:32,504 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472144 to hdfs://localhost:43879/hbase/oldWALs/wal.1731724472144 2024-11-16T02:34:32,505 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000006000 2024-11-16T02:34:32,505 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:32,508 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/test2727-manual,16010,1731724471727, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:32,526 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472508, exclude list is [], retry=0 2024-11-16T02:34:32,529 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:32,530 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:32,530 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:32,556 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731724471727/wal.1731724472508 2024-11-16T02:34:32,557 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477)] 2024-11-16T02:34:32,557 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 924a20e1b2534c0e45f0d44c6d028459, NAME => 'test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:32,557 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:32,557 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:32,557 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:32,559 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:32,560 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 924a20e1b2534c0e45f0d44c6d028459 columnFamilyName a 2024-11-16T02:34:32,560 DEBUG [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:32,561 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] regionserver.HStore(327): Store=924a20e1b2534c0e45f0d44c6d028459/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:32,561 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:32,562 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 924a20e1b2534c0e45f0d44c6d028459 columnFamilyName b 2024-11-16T02:34:32,562 DEBUG [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:32,563 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] regionserver.HStore(327): Store=924a20e1b2534c0e45f0d44c6d028459/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:32,563 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:32,564 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 924a20e1b2534c0e45f0d44c6d028459 columnFamilyName c 2024-11-16T02:34:32,564 DEBUG [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:32,565 INFO [StoreOpener-924a20e1b2534c0e45f0d44c6d028459-1 {}] regionserver.HStore(327): Store=924a20e1b2534c0e45f0d44c6d028459/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:32,565 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:32,566 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:32,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:32,569 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000003000 2024-11-16T02:34:32,606 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:32,642 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000003000 2024-11-16T02:34:32,643 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000006000 2024-11-16T02:34:32,681 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:32,716 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000006000 2024-11-16T02:34:32,717 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 924a20e1b2534c0e45f0d44c6d028459 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-16T02:34:32,736 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/.tmp/a/185b069e46074a12b7c31f0a4fb8ab7c is 41, key is test2727/a:100/1731724472202/Put/seqid=0 2024-11-16T02:34:32,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741871_1048 (size=84227) 2024-11-16T02:34:32,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741871_1048 (size=84227) 2024-11-16T02:34:32,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741871_1048 (size=84227) 2024-11-16T02:34:32,743 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/.tmp/a/185b069e46074a12b7c31f0a4fb8ab7c 2024-11-16T02:34:32,773 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/.tmp/b/0ff6f51e1f6a4f03ab7e36b88bf55456 is 41, key is test2727/b:100/1731724472257/Put/seqid=0 2024-11-16T02:34:32,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741872_1049 (size=84609) 2024-11-16T02:34:32,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741872_1049 (size=84609) 2024-11-16T02:34:32,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741872_1049 (size=84609) 2024-11-16T02:34:32,778 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/.tmp/b/0ff6f51e1f6a4f03ab7e36b88bf55456 2024-11-16T02:34:32,804 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/.tmp/c/9af4c6353e2f47a18f3e140c462c446e is 41, key is test2727/c:100/1731724472301/Put/seqid=0 2024-11-16T02:34:32,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741873_1050 (size=84609) 2024-11-16T02:34:32,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741873_1050 (size=84609) 2024-11-16T02:34:32,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741873_1050 (size=84609) 2024-11-16T02:34:32,810 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/.tmp/c/9af4c6353e2f47a18f3e140c462c446e 2024-11-16T02:34:32,817 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/.tmp/a/185b069e46074a12b7c31f0a4fb8ab7c as hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/a/185b069e46074a12b7c31f0a4fb8ab7c 2024-11-16T02:34:32,823 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/a/185b069e46074a12b7c31f0a4fb8ab7c, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-16T02:34:32,825 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/.tmp/b/0ff6f51e1f6a4f03ab7e36b88bf55456 as hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/b/0ff6f51e1f6a4f03ab7e36b88bf55456 2024-11-16T02:34:32,831 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/b/0ff6f51e1f6a4f03ab7e36b88bf55456, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-16T02:34:32,832 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/.tmp/c/9af4c6353e2f47a18f3e140c462c446e as hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/c/9af4c6353e2f47a18f3e140c462c446e 2024-11-16T02:34:32,839 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/c/9af4c6353e2f47a18f3e140c462c446e, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-16T02:34:32,839 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 924a20e1b2534c0e45f0d44c6d028459 in 122ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-16T02:34:32,840 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000003000 2024-11-16T02:34:32,841 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/0000000000000006000 2024-11-16T02:34:32,842 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:32,842 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:32,843 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:32,845 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 924a20e1b2534c0e45f0d44c6d028459 2024-11-16T02:34:32,847 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/test2727/924a20e1b2534c0e45f0d44c6d028459/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-16T02:34:32,848 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 924a20e1b2534c0e45f0d44c6d028459; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62415624, jitterRate=-0.06993472576141357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:32,849 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 924a20e1b2534c0e45f0d44c6d028459: Writing region info on filesystem at 1731724472557Initializing all the Stores at 1731724472558 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724472558Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724472559 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724472559Obtaining lock to block concurrent updates at 1731724472717 (+158 ms)Preparing flush snapshotting stores in 924a20e1b2534c0e45f0d44c6d028459 at 1731724472717Finished memstore snapshotting test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731724472717Flushing stores of test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459. at 1731724472717Flushing 924a20e1b2534c0e45f0d44c6d028459/a: creating writer at 1731724472717Flushing 924a20e1b2534c0e45f0d44c6d028459/a: appending metadata at 1731724472735 (+18 ms)Flushing 924a20e1b2534c0e45f0d44c6d028459/a: closing flushed file at 1731724472736 (+1 ms)Flushing 924a20e1b2534c0e45f0d44c6d028459/b: creating writer at 1731724472749 (+13 ms)Flushing 924a20e1b2534c0e45f0d44c6d028459/b: appending metadata at 1731724472771 (+22 ms)Flushing 924a20e1b2534c0e45f0d44c6d028459/b: closing flushed file at 1731724472771Flushing 924a20e1b2534c0e45f0d44c6d028459/c: creating writer at 1731724472784 (+13 ms)Flushing 924a20e1b2534c0e45f0d44c6d028459/c: appending metadata at 1731724472803 (+19 ms)Flushing 924a20e1b2534c0e45f0d44c6d028459/c: closing flushed file at 1731724472803Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b38a526: reopening flushed file at 1731724472816 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5938cdaf: reopening flushed file at 1731724472824 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13a7e57c: reopening flushed file at 1731724472831 (+7 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 924a20e1b2534c0e45f0d44c6d028459 in 122ms, sequenceid=6000, compaction requested=false; wal=null at 1731724472839 (+8 ms)Cleaning up temporary data from old regions at 1731724472842 (+3 ms)Region opened successfully at 1731724472849 (+7 ms) 2024-11-16T02:34:32,850 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-16T02:34:32,851 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 924a20e1b2534c0e45f0d44c6d028459, disabling compactions & flushes 2024-11-16T02:34:32,851 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459. 2024-11-16T02:34:32,851 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459. 2024-11-16T02:34:32,851 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459. after waiting 0 ms 2024-11-16T02:34:32,851 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459. 2024-11-16T02:34:32,852 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731724471729.924a20e1b2534c0e45f0d44c6d028459. 2024-11-16T02:34:32,852 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 924a20e1b2534c0e45f0d44c6d028459: Waiting for close lock at 1731724472851Disabling compacts and flushes for region at 1731724472851Disabling writes for close at 1731724472851Writing region close event to WAL at 1731724472852 (+1 ms)Closed at 1731724472852 2024-11-16T02:34:32,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741870_1047 (size=175) 2024-11-16T02:34:32,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741870_1047 (size=175) 2024-11-16T02:34:32,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741870_1047 (size=175) 2024-11-16T02:34:32,858 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:32,858 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731724472508) 2024-11-16T02:34:32,871 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=403 (was 399) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:52746 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:41312 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:34488 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=391 (was 391), ProcessCount=11 (was 11), AvailableMemoryMB=4208 (was 4444) 2024-11-16T02:34:32,884 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=403, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=391, ProcessCount=11, AvailableMemoryMB=4207 2024-11-16T02:34:32,900 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:32,907 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731724472899, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:32,908 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731724472908 2024-11-16T02:34:32,943 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731724472899/wal.1731724472908 2024-11-16T02:34:32,948 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:32,953 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 41e5c6d77a6e4ffd2f47d0f28e013d0d, NAME => 'testSequentialEditLogSeqNum,,1731724472901.41e5c6d77a6e4ffd2f47d0f28e013d0d.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:32,953 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731724472901.41e5c6d77a6e4ffd2f47d0f28e013d0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:32,953 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 41e5c6d77a6e4ffd2f47d0f28e013d0d 2024-11-16T02:34:32,953 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 41e5c6d77a6e4ffd2f47d0f28e013d0d 2024-11-16T02:34:32,954 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d doesn't exist for region: 41e5c6d77a6e4ffd2f47d0f28e013d0d on table testSequentialEditLogSeqNum 2024-11-16T02:34:32,955 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 41e5c6d77a6e4ffd2f47d0f28e013d0d on table testSequentialEditLogSeqNum 2024-11-16T02:34:32,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741875_1052 (size=62) 2024-11-16T02:34:32,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741875_1052 (size=62) 2024-11-16T02:34:32,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741875_1052 (size=62) 2024-11-16T02:34:32,969 INFO [StoreOpener-41e5c6d77a6e4ffd2f47d0f28e013d0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41e5c6d77a6e4ffd2f47d0f28e013d0d 2024-11-16T02:34:32,971 INFO [StoreOpener-41e5c6d77a6e4ffd2f47d0f28e013d0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41e5c6d77a6e4ffd2f47d0f28e013d0d columnFamilyName a 2024-11-16T02:34:32,971 DEBUG [StoreOpener-41e5c6d77a6e4ffd2f47d0f28e013d0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:32,971 INFO [StoreOpener-41e5c6d77a6e4ffd2f47d0f28e013d0d-1 {}] regionserver.HStore(327): Store=41e5c6d77a6e4ffd2f47d0f28e013d0d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:32,972 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41e5c6d77a6e4ffd2f47d0f28e013d0d 2024-11-16T02:34:32,973 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d 2024-11-16T02:34:32,973 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d 2024-11-16T02:34:32,973 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41e5c6d77a6e4ffd2f47d0f28e013d0d 2024-11-16T02:34:32,974 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41e5c6d77a6e4ffd2f47d0f28e013d0d 2024-11-16T02:34:32,976 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41e5c6d77a6e4ffd2f47d0f28e013d0d 2024-11-16T02:34:32,978 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:32,979 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41e5c6d77a6e4ffd2f47d0f28e013d0d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66729812, jitterRate=-0.005648314952850342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T02:34:32,980 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41e5c6d77a6e4ffd2f47d0f28e013d0d: Writing region info on filesystem at 1731724472953Initializing all the Stores at 1731724472969 (+16 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724472969Cleaning up temporary data from old regions at 1731724472974 (+5 ms)Region opened successfully at 1731724472979 (+5 ms) 2024-11-16T02:34:32,993 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 41e5c6d77a6e4ffd2f47d0f28e013d0d 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-16T02:34:33,014 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/.tmp/a/037f5e0b208f450d9d49c51e995a8934 is 81, key is testSequentialEditLogSeqNum/a:x0/1731724472980/Put/seqid=0 2024-11-16T02:34:33,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741876_1053 (size=5833) 2024-11-16T02:34:33,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741876_1053 (size=5833) 2024-11-16T02:34:33,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741876_1053 (size=5833) 2024-11-16T02:34:33,024 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/.tmp/a/037f5e0b208f450d9d49c51e995a8934 2024-11-16T02:34:33,031 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/.tmp/a/037f5e0b208f450d9d49c51e995a8934 as hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/a/037f5e0b208f450d9d49c51e995a8934 2024-11-16T02:34:33,039 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/a/037f5e0b208f450d9d49c51e995a8934, entries=10, sequenceid=13, filesize=5.7 K 2024-11-16T02:34:33,040 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 41e5c6d77a6e4ffd2f47d0f28e013d0d in 47ms, sequenceid=13, compaction requested=false 2024-11-16T02:34:33,040 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 41e5c6d77a6e4ffd2f47d0f28e013d0d: 2024-11-16T02:34:33,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-16T02:34:33,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-16T02:34:33,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-16T02:34:33,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-16T02:34:33,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-16T02:34:33,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741874_1051 (size=3256) 2024-11-16T02:34:33,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741874_1051 (size=3256) 2024-11-16T02:34:33,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741874_1051 (size=3256) 2024-11-16T02:34:33,074 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731724472899/wal.1731724472908, size=3.2 K (3256bytes) 2024-11-16T02:34:33,074 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731724472899/wal.1731724472908 2024-11-16T02:34:33,075 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731724472899/wal.1731724472908 after 1ms 2024-11-16T02:34:33,111 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:33,111 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731724472899/wal.1731724472908 took 37ms 2024-11-16T02:34:33,114 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731724472899/wal.1731724472908 so closing down 2024-11-16T02:34:33,114 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:33,115 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731724472908.temp 2024-11-16T02:34:33,154 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/recovered.edits/0000000000000000003-wal.1731724472908.temp 2024-11-16T02:34:33,155 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:33,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741877_1054 (size=2650) 2024-11-16T02:34:33,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741877_1054 (size=2650) 2024-11-16T02:34:33,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741877_1054 (size=2650) 2024-11-16T02:34:33,163 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/recovered.edits/0000000000000000003-wal.1731724472908.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-16T02:34:33,164 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/recovered.edits/0000000000000000003-wal.1731724472908.temp to hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/recovered.edits/0000000000000000020 2024-11-16T02:34:33,164 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 53 ms; skipped=2; WAL=hdfs://localhost:43879/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731724472899/wal.1731724472908, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-16T02:34:33,164 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731724472899/wal.1731724472908, journal: Splitting hdfs://localhost:43879/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731724472899/wal.1731724472908, size=3.2 K (3256bytes) at 1731724473074Finishing writing output for hdfs://localhost:43879/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731724472899/wal.1731724472908 so closing down at 1731724473114 (+40 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/recovered.edits/0000000000000000003-wal.1731724472908.temp at 1731724473154 (+40 ms)3 split writer threads finished at 1731724473155 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/recovered.edits/0000000000000000003-wal.1731724472908.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731724473163 (+8 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/recovered.edits/0000000000000000003-wal.1731724472908.temp to hdfs://localhost:43879/hbase/data/default/testSequentialEditLogSeqNum/41e5c6d77a6e4ffd2f47d0f28e013d0d/recovered.edits/0000000000000000020 at 1731724473164 (+1 ms)Processed 17 edits across 1 Regions in 53 ms; skipped=2; WAL=hdfs://localhost:43879/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731724472899/wal.1731724472908, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1731724473164 2024-11-16T02:34:33,179 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=408 (was 403) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:41312 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:34488 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=391 (was 391), ProcessCount=11 (was 11), AvailableMemoryMB=4199 (was 4207) 2024-11-16T02:34:33,194 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=408, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=391, ProcessCount=11, AvailableMemoryMB=4198 2024-11-16T02:34:33,214 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:33,216 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:33,245 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:33,248 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39739351, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/hregion-39739351, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:33,261 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39739351/hregion-39739351.1731724473249, exclude list is [], retry=0 2024-11-16T02:34:33,265 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:33,265 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:33,266 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:33,306 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39739351/hregion-39739351.1731724473249 2024-11-16T02:34:33,306 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:33,307 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9840286fdfddf8deb0e2cee19b7882cd, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43879/hbase 2024-11-16T02:34:33,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741879_1056 (size=70) 2024-11-16T02:34:33,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741879_1056 (size=70) 2024-11-16T02:34:33,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741879_1056 (size=70) 2024-11-16T02:34:33,320 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:33,321 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,323 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9840286fdfddf8deb0e2cee19b7882cd columnFamilyName a 2024-11-16T02:34:33,323 DEBUG [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:33,324 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(327): Store=9840286fdfddf8deb0e2cee19b7882cd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:33,324 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,325 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9840286fdfddf8deb0e2cee19b7882cd columnFamilyName b 2024-11-16T02:34:33,325 DEBUG [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:33,326 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(327): Store=9840286fdfddf8deb0e2cee19b7882cd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:33,326 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,328 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9840286fdfddf8deb0e2cee19b7882cd columnFamilyName c 2024-11-16T02:34:33,328 DEBUG [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:33,328 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(327): Store=9840286fdfddf8deb0e2cee19b7882cd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:33,329 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,329 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,330 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,331 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,331 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,331 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:33,332 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,334 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:33,335 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9840286fdfddf8deb0e2cee19b7882cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69580387, jitterRate=0.03682856261730194}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:33,335 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9840286fdfddf8deb0e2cee19b7882cd: Writing region info on filesystem at 1731724473320Initializing all the Stores at 1731724473321 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724473321Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724473321Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724473321Cleaning up temporary data from old regions at 1731724473331 (+10 ms)Region opened successfully at 1731724473335 (+4 ms) 2024-11-16T02:34:33,336 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9840286fdfddf8deb0e2cee19b7882cd, disabling compactions & flushes 2024-11-16T02:34:33,336 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd. 2024-11-16T02:34:33,336 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd. 2024-11-16T02:34:33,336 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd. after waiting 0 ms 2024-11-16T02:34:33,336 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd. 2024-11-16T02:34:33,336 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd. 2024-11-16T02:34:33,336 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9840286fdfddf8deb0e2cee19b7882cd: Waiting for close lock at 1731724473336Disabling compacts and flushes for region at 1731724473336Disabling writes for close at 1731724473336Writing region close event to WAL at 1731724473336Closed at 1731724473336 2024-11-16T02:34:33,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741878_1055 (size=175) 2024-11-16T02:34:33,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741878_1055 (size=175) 2024-11-16T02:34:33,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741878_1055 (size=175) 2024-11-16T02:34:33,342 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:33,342 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39739351:(num 1731724473249) 2024-11-16T02:34:33,342 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:33,344 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:33,357 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344, exclude list is [], retry=0 2024-11-16T02:34:33,360 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:33,360 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:33,360 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:33,388 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344 2024-11-16T02:34:33,388 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131)] 2024-11-16T02:34:33,389 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9840286fdfddf8deb0e2cee19b7882cd, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:33,389 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:33,389 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,389 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,391 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,392 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9840286fdfddf8deb0e2cee19b7882cd columnFamilyName a 2024-11-16T02:34:33,392 DEBUG [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:33,393 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(327): Store=9840286fdfddf8deb0e2cee19b7882cd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:33,393 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,394 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9840286fdfddf8deb0e2cee19b7882cd columnFamilyName b 2024-11-16T02:34:33,395 DEBUG [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:33,396 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(327): Store=9840286fdfddf8deb0e2cee19b7882cd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:33,396 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,397 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9840286fdfddf8deb0e2cee19b7882cd columnFamilyName c 2024-11-16T02:34:33,397 DEBUG [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:33,398 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(327): Store=9840286fdfddf8deb0e2cee19b7882cd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:33,398 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,399 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,400 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,402 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,402 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,402 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:33,405 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:33,406 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9840286fdfddf8deb0e2cee19b7882cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62181662, jitterRate=-0.07342103123664856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:33,407 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9840286fdfddf8deb0e2cee19b7882cd: Writing region info on filesystem at 1731724473389Initializing all the Stores at 1731724473390 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724473390Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724473391 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724473391Cleaning up temporary data from old regions at 1731724473402 (+11 ms)Region opened successfully at 1731724473407 (+5 ms) 2024-11-16T02:34:33,413 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731724473411/Put/seqid=0 2024-11-16T02:34:33,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741881_1058 (size=4826) 2024-11-16T02:34:33,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741881_1058 (size=4826) 2024-11-16T02:34:33,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741881_1058 (size=4826) 2024-11-16T02:34:33,421 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43879/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 9840286fdfddf8deb0e2cee19b7882cd/a 2024-11-16T02:34:33,429 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-16T02:34:33,429 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-16T02:34:33,429 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9840286fdfddf8deb0e2cee19b7882cd: 2024-11-16T02:34:33,431 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/a/2f733673a0ba4d93a8e0914f17ba25d4_SeqId_3_ 2024-11-16T02:34:33,432 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43879/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 9840286fdfddf8deb0e2cee19b7882cd/a as hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/a/2f733673a0ba4d93a8e0914f17ba25d4_SeqId_3_ - updating store file list. 2024-11-16T02:34:33,439 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2f733673a0ba4d93a8e0914f17ba25d4_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-16T02:34:33,440 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/a/2f733673a0ba4d93a8e0914f17ba25d4_SeqId_3_ into 9840286fdfddf8deb0e2cee19b7882cd/a 2024-11-16T02:34:33,440 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43879/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 9840286fdfddf8deb0e2cee19b7882cd/a (new location: hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/a/2f733673a0ba4d93a8e0914f17ba25d4_SeqId_3_) 2024-11-16T02:34:33,482 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344, size=0 (0bytes) 2024-11-16T02:34:33,482 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344 might be still open, length is 0 2024-11-16T02:34:33,482 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344 2024-11-16T02:34:33,482 WARN [IPC Server handler 0 on default port 43879 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-16T02:34:33,483 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344 after 1ms 2024-11-16T02:34:35,643 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:41402 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41402 dst: /127.0.0.1:40973 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40973 remote=/127.0.0.1:41402]. Total timeout mills is 60000, 57804 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:35,645 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:59268 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39531:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59268 dst: /127.0.0.1:39531 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:35,645 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:34566 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33993:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34566 dst: /127.0.0.1:33993 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:35,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741880_1059 (size=606) 2024-11-16T02:34:35,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741880_1059 (size=606) 2024-11-16T02:34:37,484 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344 after 4002ms 2024-11-16T02:34:37,519 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:37,519 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344 took 4037ms 2024-11-16T02:34:37,521 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344; continuing. 2024-11-16T02:34:37,521 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344 so closing down 2024-11-16T02:34:37,521 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:37,523 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731724473344.temp 2024-11-16T02:34:37,546 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/0000000000000000005-wal.1731724473344.temp 2024-11-16T02:34:37,546 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:37,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741882_1060 (size=329) 2024-11-16T02:34:37,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741882_1060 (size=329) 2024-11-16T02:34:37,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741882_1060 (size=329) 2024-11-16T02:34:37,553 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/0000000000000000005-wal.1731724473344.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T02:34:37,555 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/0000000000000000005-wal.1731724473344.temp to hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/0000000000000000005 2024-11-16T02:34:37,555 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344, size=0, length=0, corrupted=false, cancelled=false 2024-11-16T02:34:37,555 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344, journal: Splitting hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344, size=0 (0bytes) at 1731724473482Finishing writing output for hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344 so closing down at 1731724477521 (+4039 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/0000000000000000005-wal.1731724473344.temp at 1731724477546 (+25 ms)3 split writer threads finished at 1731724477546Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/0000000000000000005-wal.1731724473344.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731724477554 (+8 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/0000000000000000005-wal.1731724473344.temp to hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/0000000000000000005 at 1731724477555 (+1 ms)Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344, size=0, length=0, corrupted=false, cancelled=false at 1731724477555 2024-11-16T02:34:37,557 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344 to hdfs://localhost:43879/hbase/oldWALs/wal.1731724473344 2024-11-16T02:34:37,558 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/0000000000000000005 2024-11-16T02:34:37,558 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:37,560 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:37,573 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724477561, exclude list is [], retry=0 2024-11-16T02:34:37,575 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:37,576 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:37,576 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:37,599 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724477561 2024-11-16T02:34:37,600 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:38477:38477)] 2024-11-16T02:34:37,600 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9840286fdfddf8deb0e2cee19b7882cd, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:37,600 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:37,600 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:37,600 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:37,602 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:37,603 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9840286fdfddf8deb0e2cee19b7882cd columnFamilyName a 2024-11-16T02:34:37,603 DEBUG [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:37,609 DEBUG [StoreFileOpener-9840286fdfddf8deb0e2cee19b7882cd-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2f733673a0ba4d93a8e0914f17ba25d4_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-16T02:34:37,609 DEBUG [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/a/2f733673a0ba4d93a8e0914f17ba25d4_SeqId_3_ 2024-11-16T02:34:37,610 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(327): Store=9840286fdfddf8deb0e2cee19b7882cd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:37,610 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:37,611 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9840286fdfddf8deb0e2cee19b7882cd columnFamilyName b 2024-11-16T02:34:37,611 DEBUG [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:37,611 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(327): Store=9840286fdfddf8deb0e2cee19b7882cd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:37,612 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:37,612 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9840286fdfddf8deb0e2cee19b7882cd columnFamilyName c 2024-11-16T02:34:37,613 DEBUG [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:37,613 INFO [StoreOpener-9840286fdfddf8deb0e2cee19b7882cd-1 {}] regionserver.HStore(327): Store=9840286fdfddf8deb0e2cee19b7882cd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:37,613 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:37,614 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:37,616 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:37,616 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/0000000000000000005 2024-11-16T02:34:37,643 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:37,644 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/0000000000000000005 2024-11-16T02:34:37,644 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9840286fdfddf8deb0e2cee19b7882cd 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-16T02:34:37,658 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/.tmp/a/8a88bfb6035246c4b594f622ab5c42a2 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731724473446/Put/seqid=0 2024-11-16T02:34:37,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741884_1062 (size=5149) 2024-11-16T02:34:37,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741884_1062 (size=5149) 2024-11-16T02:34:37,664 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/.tmp/a/8a88bfb6035246c4b594f622ab5c42a2 2024-11-16T02:34:37,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741884_1062 (size=5149) 2024-11-16T02:34:37,671 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/.tmp/a/8a88bfb6035246c4b594f622ab5c42a2 as hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/a/8a88bfb6035246c4b594f622ab5c42a2 2024-11-16T02:34:37,676 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/a/8a88bfb6035246c4b594f622ab5c42a2, entries=1, sequenceid=5, filesize=5.0 K 2024-11-16T02:34:37,677 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 9840286fdfddf8deb0e2cee19b7882cd in 33ms, sequenceid=5, compaction requested=false; wal=null 2024-11-16T02:34:37,677 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/0000000000000000005 2024-11-16T02:34:37,678 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:37,678 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:37,679 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:37,680 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9840286fdfddf8deb0e2cee19b7882cd 2024-11-16T02:34:37,682 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9840286fdfddf8deb0e2cee19b7882cd/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-16T02:34:37,683 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9840286fdfddf8deb0e2cee19b7882cd; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74240271, jitterRate=0.10626624524593353}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:37,683 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9840286fdfddf8deb0e2cee19b7882cd: Writing region info on filesystem at 1731724477600Initializing all the Stores at 1731724477601 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724477601Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724477601Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724477602 (+1 ms)Obtaining lock to block concurrent updates at 1731724477644 (+42 ms)Preparing flush snapshotting stores in 9840286fdfddf8deb0e2cee19b7882cd at 1731724477644Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731724477644Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd. at 1731724477644Flushing 9840286fdfddf8deb0e2cee19b7882cd/a: creating writer at 1731724477644Flushing 9840286fdfddf8deb0e2cee19b7882cd/a: appending metadata at 1731724477657 (+13 ms)Flushing 9840286fdfddf8deb0e2cee19b7882cd/a: closing flushed file at 1731724477657Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11cde2c0: reopening flushed file at 1731724477670 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 9840286fdfddf8deb0e2cee19b7882cd in 33ms, sequenceid=5, compaction requested=false; wal=null at 1731724477677 (+7 ms)Cleaning up temporary data from old regions at 1731724477679 (+2 ms)Region opened successfully at 1731724477683 (+4 ms) 2024-11-16T02:34:37,687 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9840286fdfddf8deb0e2cee19b7882cd, disabling compactions & flushes 2024-11-16T02:34:37,687 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd. 2024-11-16T02:34:37,687 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd. 2024-11-16T02:34:37,687 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd. after waiting 0 ms 2024-11-16T02:34:37,687 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd. 2024-11-16T02:34:37,688 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731724473214.9840286fdfddf8deb0e2cee19b7882cd. 2024-11-16T02:34:37,688 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9840286fdfddf8deb0e2cee19b7882cd: Waiting for close lock at 1731724477687Disabling compacts and flushes for region at 1731724477687Disabling writes for close at 1731724477687Writing region close event to WAL at 1731724477688 (+1 ms)Closed at 1731724477688 2024-11-16T02:34:37,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741883_1061 (size=175) 2024-11-16T02:34:37,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741883_1061 (size=175) 2024-11-16T02:34:37,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741883_1061 (size=175) 2024-11-16T02:34:37,694 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:37,694 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731724477561) 2024-11-16T02:34:37,707 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=411 (was 408) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40745 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1007598937_22 at /127.0.0.1:34606 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1007598937_22 at /127.0.0.1:41438 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:40745 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1007598937_22 at /127.0.0.1:59284 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:43879 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:43879 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 391), ProcessCount=11 (was 11), AvailableMemoryMB=4203 (was 4198) - AvailableMemoryMB LEAK? - 2024-11-16T02:34:37,708 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-16T02:34:37,720 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=411, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=4202 2024-11-16T02:34:37,721 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-16T02:34:37,738 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:37,742 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T02:34:37,745 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 667222f7cf1b,38401,1731724461545 2024-11-16T02:34:37,747 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63f843be 2024-11-16T02:34:37,748 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T02:34:37,750 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T02:34:37,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T02:34:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-16T02:34:37,761 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T02:34:37,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-16T02:34:37,763 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:37,765 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T02:34:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T02:34:37,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741885_1063 (size=694) 2024-11-16T02:34:37,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741885_1063 (size=694) 2024-11-16T02:34:37,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741885_1063 (size=694) 2024-11-16T02:34:37,777 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 056b939bd43275b5084f24034a22ab92, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9 2024-11-16T02:34:37,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741886_1064 (size=77) 2024-11-16T02:34:37,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741886_1064 (size=77) 2024-11-16T02:34:37,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741886_1064 (size=77) 2024-11-16T02:34:37,786 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:37,786 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 056b939bd43275b5084f24034a22ab92, disabling compactions & flushes 2024-11-16T02:34:37,786 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:37,786 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:37,786 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. after waiting 0 ms 2024-11-16T02:34:37,786 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:37,786 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:37,786 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 056b939bd43275b5084f24034a22ab92: Waiting for close lock at 1731724477786Disabling compacts and flushes for region at 1731724477786Disabling writes for close at 1731724477786Writing region close event to WAL at 1731724477786Closed at 1731724477786 2024-11-16T02:34:37,788 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T02:34:37,793 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731724477788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731724477788"}]},"ts":"1731724477788"} 2024-11-16T02:34:37,796 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T02:34:37,798 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T02:34:37,800 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731724477798"}]},"ts":"1731724477798"} 2024-11-16T02:34:37,807 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-16T02:34:37,807 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {667222f7cf1b=0} racks are {/default-rack=0} 2024-11-16T02:34:37,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T02:34:37,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T02:34:37,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-16T02:34:37,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T02:34:37,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T02:34:37,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T02:34:37,809 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T02:34:37,809 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T02:34:37,809 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T02:34:37,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T02:34:37,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, ASSIGN}] 2024-11-16T02:34:37,813 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, ASSIGN 2024-11-16T02:34:37,815 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, ASSIGN; state=OFFLINE, location=667222f7cf1b,40209,1731724462418; forceNewPlan=false, retain=false 2024-11-16T02:34:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T02:34:37,973 INFO [667222f7cf1b:38401 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T02:34:37,976 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=056b939bd43275b5084f24034a22ab92, regionState=OPENING, regionLocation=667222f7cf1b,40209,1731724462418 2024-11-16T02:34:37,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, ASSIGN because future has completed 2024-11-16T02:34:37,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40209,1731724462418}] 2024-11-16T02:34:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T02:34:38,133 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T02:34:38,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59833, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T02:34:38,145 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:38,145 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 056b939bd43275b5084f24034a22ab92, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:38,146 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,146 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:38,146 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,146 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,148 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,150 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056b939bd43275b5084f24034a22ab92 columnFamilyName cf1 2024-11-16T02:34:38,150 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:38,151 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(327): Store=056b939bd43275b5084f24034a22ab92/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:38,151 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,153 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056b939bd43275b5084f24034a22ab92 columnFamilyName cf2 2024-11-16T02:34:38,153 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:38,154 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(327): Store=056b939bd43275b5084f24034a22ab92/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:38,154 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,155 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,156 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,157 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,157 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,159 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-16T02:34:38,161 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,165 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:38,166 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 056b939bd43275b5084f24034a22ab92; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74113043, jitterRate=0.10437040030956268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-16T02:34:38,166 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,167 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 056b939bd43275b5084f24034a22ab92: Running coprocessor pre-open hook at 1731724478146Writing region info on filesystem at 1731724478146Initializing all the Stores at 1731724478148 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724478148Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724478148Cleaning up temporary data from old regions at 1731724478157 (+9 ms)Running coprocessor post-open hooks at 1731724478166 (+9 ms)Region opened successfully at 1731724478167 (+1 ms) 2024-11-16T02:34:38,170 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., pid=6, masterSystemTime=1731724478133 2024-11-16T02:34:38,173 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:38,173 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:38,175 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=056b939bd43275b5084f24034a22ab92, regionState=OPEN, openSeqNum=2, regionLocation=667222f7cf1b,40209,1731724462418 2024-11-16T02:34:38,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40209,1731724462418 because future has completed 2024-11-16T02:34:38,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T02:34:38,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40209,1731724462418 in 202 msec 2024-11-16T02:34:38,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T02:34:38,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, ASSIGN in 374 msec 2024-11-16T02:34:38,191 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T02:34:38,192 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731724478191"}]},"ts":"1731724478191"} 2024-11-16T02:34:38,194 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-16T02:34:38,196 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T02:34:38,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 441 msec 2024-11-16T02:34:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T02:34:38,399 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-16T02:34:38,399 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-16T02:34:38,403 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T02:34:38,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-16T02:34:38,413 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T02:34:38,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-16T02:34:38,425 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=2] 2024-11-16T02:34:38,426 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T02:34:38,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T02:34:38,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=056b939bd43275b5084f24034a22ab92, source=667222f7cf1b,40209,1731724462418, destination=667222f7cf1b,40547,1731724462323, warming up region on 667222f7cf1b,40547,1731724462323 2024-11-16T02:34:38,447 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=056b939bd43275b5084f24034a22ab92, source=667222f7cf1b,40209,1731724462418, destination=667222f7cf1b,40547,1731724462323, running balancer 2024-11-16T02:34:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, REOPEN/MOVE 2024-11-16T02:34:38,449 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, REOPEN/MOVE 2024-11-16T02:34:38,451 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=056b939bd43275b5084f24034a22ab92, regionState=CLOSING, regionLocation=667222f7cf1b,40209,1731724462418 2024-11-16T02:34:38,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40547 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40547 {}] regionserver.HRegion(7855): Warmup {ENCODED => 056b939bd43275b5084f24034a22ab92, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40547 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:38,453 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,454 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056b939bd43275b5084f24034a22ab92 columnFamilyName cf1 2024-11-16T02:34:38,454 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:38,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, REOPEN/MOVE because future has completed 2024-11-16T02:34:38,455 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(327): Store=056b939bd43275b5084f24034a22ab92/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:38,455 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,456 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056b939bd43275b5084f24034a22ab92 columnFamilyName cf2 2024-11-16T02:34:38,456 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T02:34:38,456 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:38,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40209,1731724462418}] 2024-11-16T02:34:38,457 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(327): Store=056b939bd43275b5084f24034a22ab92/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40547 {}] regionserver.HRegion(1722): Closing 056b939bd43275b5084f24034a22ab92, disabling compactions & flushes 2024-11-16T02:34:38,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40547 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40547 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40547 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. after waiting 0 ms 2024-11-16T02:34:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40547 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:38,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40547 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40547 {}] regionserver.HRegion(1676): Region close journal for 056b939bd43275b5084f24034a22ab92: Waiting for close lock at 1731724478457Disabling compacts and flushes for region at 1731724478457Disabling writes for close at 1731724478457Writing region close event to WAL at 1731724478458 (+1 ms)Closed at 1731724478458 2024-11-16T02:34:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-16T02:34:38,614 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,614 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T02:34:38,614 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 056b939bd43275b5084f24034a22ab92, disabling compactions & flushes 2024-11-16T02:34:38,615 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:38,615 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:38,615 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. after waiting 0 ms 2024-11-16T02:34:38,615 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:38,615 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 056b939bd43275b5084f24034a22ab92 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-16T02:34:38,632 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp/cf1/3ba8bd5871ff42d2aa8c98097ddb0a24 is 35, key is r1/cf1:q/1731724478429/Put/seqid=0 2024-11-16T02:34:38,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741887_1065 (size=4783) 2024-11-16T02:34:38,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741887_1065 (size=4783) 2024-11-16T02:34:38,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741887_1065 (size=4783) 2024-11-16T02:34:38,646 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp/cf1/3ba8bd5871ff42d2aa8c98097ddb0a24 2024-11-16T02:34:38,655 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp/cf1/3ba8bd5871ff42d2aa8c98097ddb0a24 as hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/3ba8bd5871ff42d2aa8c98097ddb0a24 2024-11-16T02:34:38,662 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/3ba8bd5871ff42d2aa8c98097ddb0a24, entries=1, sequenceid=5, filesize=4.7 K 2024-11-16T02:34:38,663 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 056b939bd43275b5084f24034a22ab92 in 48ms, sequenceid=5, compaction requested=false 2024-11-16T02:34:38,663 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-16T02:34:38,669 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T02:34:38,671 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:38,672 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 056b939bd43275b5084f24034a22ab92: Waiting for close lock at 1731724478614Running coprocessor pre-close hooks at 1731724478614Disabling compacts and flushes for region at 1731724478614Disabling writes for close at 1731724478615 (+1 ms)Obtaining lock to block concurrent updates at 1731724478615Preparing flush snapshotting stores in 056b939bd43275b5084f24034a22ab92 at 1731724478615Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731724478615Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. at 1731724478616 (+1 ms)Flushing 056b939bd43275b5084f24034a22ab92/cf1: creating writer at 1731724478616Flushing 056b939bd43275b5084f24034a22ab92/cf1: appending metadata at 1731724478632 (+16 ms)Flushing 056b939bd43275b5084f24034a22ab92/cf1: closing flushed file at 1731724478632Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35655813: reopening flushed file at 1731724478653 (+21 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 056b939bd43275b5084f24034a22ab92 in 48ms, sequenceid=5, compaction requested=false at 1731724478663 (+10 ms)Writing region close event to WAL at 1731724478665 (+2 ms)Running coprocessor post-close hooks at 1731724478670 (+5 ms)Closed at 1731724478671 (+1 ms) 2024-11-16T02:34:38,672 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 056b939bd43275b5084f24034a22ab92 move to 667222f7cf1b,40547,1731724462323 record at close sequenceid=5 2024-11-16T02:34:38,675 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:38,676 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=056b939bd43275b5084f24034a22ab92, regionState=CLOSED 2024-11-16T02:34:38,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40209,1731724462418 because future has completed 2024-11-16T02:34:38,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T02:34:38,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40209,1731724462418 in 225 msec 2024-11-16T02:34:38,686 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, REOPEN/MOVE; state=CLOSED, location=667222f7cf1b,40547,1731724462323; forceNewPlan=false, retain=false 2024-11-16T02:34:38,836 INFO [667222f7cf1b:38401 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T02:34:38,837 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=056b939bd43275b5084f24034a22ab92, regionState=OPENING, regionLocation=667222f7cf1b,40547,1731724462323 2024-11-16T02:34:38,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, REOPEN/MOVE because future has completed 2024-11-16T02:34:38,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40547,1731724462323}] 2024-11-16T02:34:39,002 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,003 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 056b939bd43275b5084f24034a22ab92, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:39,003 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,003 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:39,004 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,004 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,006 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,008 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056b939bd43275b5084f24034a22ab92 columnFamilyName cf1 2024-11-16T02:34:39,008 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:39,017 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/3ba8bd5871ff42d2aa8c98097ddb0a24 2024-11-16T02:34:39,017 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(327): Store=056b939bd43275b5084f24034a22ab92/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:39,017 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,018 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056b939bd43275b5084f24034a22ab92 columnFamilyName cf2 2024-11-16T02:34:39,018 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:39,019 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(327): Store=056b939bd43275b5084f24034a22ab92/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:39,019 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,020 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,022 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,023 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,023 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,023 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-16T02:34:39,025 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,026 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 056b939bd43275b5084f24034a22ab92; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65254722, jitterRate=-0.02762886881828308}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-16T02:34:39,026 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,027 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 056b939bd43275b5084f24034a22ab92: Running coprocessor pre-open hook at 1731724479004Writing region info on filesystem at 1731724479004Initializing all the Stores at 1731724479006 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724479006Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724479006Cleaning up temporary data from old regions at 1731724479023 (+17 ms)Running coprocessor post-open hooks at 1731724479026 (+3 ms)Region opened successfully at 1731724479027 (+1 ms) 2024-11-16T02:34:39,028 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., pid=9, masterSystemTime=1731724478994 2024-11-16T02:34:39,030 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,031 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=056b939bd43275b5084f24034a22ab92, regionState=OPEN, openSeqNum=9, regionLocation=667222f7cf1b,40547,1731724462323 2024-11-16T02:34:39,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40547,1731724462323 because future has completed 2024-11-16T02:34:39,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-16T02:34:39,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40547,1731724462323 in 195 msec 2024-11-16T02:34:39,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, REOPEN/MOVE in 593 msec 2024-11-16T02:34:39,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T02:34:39,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T02:34:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:49848 deadline: 1731724539061, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=667222f7cf1b port=40547 startCode=1731724462323. As of locationSeqNum=5. 2024-11-16T02:34:39,093 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=667222f7cf1b port=40547 startCode=1731724462323. As of locationSeqNum=5. 2024-11-16T02:34:39,094 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=667222f7cf1b port=40547 startCode=1731724462323. As of locationSeqNum=5. 2024-11-16T02:34:39,094 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40547,1731724462323, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=667222f7cf1b port=40547 startCode=1731724462323. As of locationSeqNum=5. 2024-11-16T02:34:39,215 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 056b939bd43275b5084f24034a22ab92 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-16T02:34:39,231 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp/cf1/54eeae3876c3418ea1cbcfb1b88e72c9 is 29, key is r1/cf1:/1731724479207/DeleteFamily/seqid=0 2024-11-16T02:34:39,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741888_1066 (size=4906) 2024-11-16T02:34:39,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741888_1066 (size=4906) 2024-11-16T02:34:39,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741888_1066 (size=4906) 2024-11-16T02:34:39,239 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp/cf1/54eeae3876c3418ea1cbcfb1b88e72c9 2024-11-16T02:34:39,245 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 54eeae3876c3418ea1cbcfb1b88e72c9 2024-11-16T02:34:39,263 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp/cf2/d4fbca7a4801448ca5da7ba1407486c5 is 29, key is r1/cf2:/1731724479207/DeleteFamily/seqid=0 2024-11-16T02:34:39,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741889_1067 (size=4906) 2024-11-16T02:34:39,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741889_1067 (size=4906) 2024-11-16T02:34:39,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741889_1067 (size=4906) 2024-11-16T02:34:39,270 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp/cf2/d4fbca7a4801448ca5da7ba1407486c5 2024-11-16T02:34:39,276 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d4fbca7a4801448ca5da7ba1407486c5 2024-11-16T02:34:39,277 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp/cf1/54eeae3876c3418ea1cbcfb1b88e72c9 as hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/54eeae3876c3418ea1cbcfb1b88e72c9 2024-11-16T02:34:39,283 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 54eeae3876c3418ea1cbcfb1b88e72c9 2024-11-16T02:34:39,283 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/54eeae3876c3418ea1cbcfb1b88e72c9, entries=1, sequenceid=12, filesize=4.8 K 2024-11-16T02:34:39,284 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp/cf2/d4fbca7a4801448ca5da7ba1407486c5 as hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf2/d4fbca7a4801448ca5da7ba1407486c5 2024-11-16T02:34:39,290 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d4fbca7a4801448ca5da7ba1407486c5 2024-11-16T02:34:39,291 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf2/d4fbca7a4801448ca5da7ba1407486c5, entries=1, sequenceid=12, filesize=4.8 K 2024-11-16T02:34:39,292 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 056b939bd43275b5084f24034a22ab92 in 77ms, sequenceid=12, compaction requested=false 2024-11-16T02:34:39,292 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 056b939bd43275b5084f24034a22ab92: 2024-11-16T02:34:39,295 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T02:34:39,296 DEBUG [Time-limited test {}] regionserver.HStore(1541): 056b939bd43275b5084f24034a22ab92/cf1 is initiating major compaction (all files) 2024-11-16T02:34:39,296 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T02:34:39,296 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:39,296 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 056b939bd43275b5084f24034a22ab92/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,297 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/3ba8bd5871ff42d2aa8c98097ddb0a24, hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/54eeae3876c3418ea1cbcfb1b88e72c9] into tmpdir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp, totalSize=9.5 K 2024-11-16T02:34:39,298 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3ba8bd5871ff42d2aa8c98097ddb0a24, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731724478429 2024-11-16T02:34:39,298 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 54eeae3876c3418ea1cbcfb1b88e72c9, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-16T02:34:39,310 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 056b939bd43275b5084f24034a22ab92#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T02:34:39,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741890_1068 (size=4626) 2024-11-16T02:34:39,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741890_1068 (size=4626) 2024-11-16T02:34:39,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741890_1068 (size=4626) 2024-11-16T02:34:39,323 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp/cf1/1494e7e13b0f4f7cbd7100b7879bc0d0 as hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/1494e7e13b0f4f7cbd7100b7879bc0d0 2024-11-16T02:34:39,336 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 056b939bd43275b5084f24034a22ab92/cf1 of 056b939bd43275b5084f24034a22ab92 into 1494e7e13b0f4f7cbd7100b7879bc0d0(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T02:34:39,336 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 056b939bd43275b5084f24034a22ab92: 2024-11-16T02:34:39,336 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-16T02:34:39,337 DEBUG [Time-limited test {}] regionserver.HStore(1541): 056b939bd43275b5084f24034a22ab92/cf2 is initiating major compaction (all files) 2024-11-16T02:34:39,337 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T02:34:39,337 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T02:34:39,337 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 056b939bd43275b5084f24034a22ab92/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,337 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf2/d4fbca7a4801448ca5da7ba1407486c5] into tmpdir=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp, totalSize=4.8 K 2024-11-16T02:34:39,338 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d4fbca7a4801448ca5da7ba1407486c5, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-16T02:34:39,345 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 056b939bd43275b5084f24034a22ab92#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T02:34:39,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741891_1069 (size=4592) 2024-11-16T02:34:39,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741891_1069 (size=4592) 2024-11-16T02:34:39,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741891_1069 (size=4592) 2024-11-16T02:34:39,367 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/.tmp/cf2/491810076f2c4278a156c746cfa3f212 as hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf2/491810076f2c4278a156c746cfa3f212 2024-11-16T02:34:39,375 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 056b939bd43275b5084f24034a22ab92/cf2 of 056b939bd43275b5084f24034a22ab92 into 491810076f2c4278a156c746cfa3f212(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T02:34:39,375 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 056b939bd43275b5084f24034a22ab92: 2024-11-16T02:34:39,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=056b939bd43275b5084f24034a22ab92, source=667222f7cf1b,40547,1731724462323, destination=667222f7cf1b,40209,1731724462418, warming up region on 667222f7cf1b,40209,1731724462418 2024-11-16T02:34:39,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=056b939bd43275b5084f24034a22ab92, source=667222f7cf1b,40547,1731724462323, destination=667222f7cf1b,40209,1731724462418, running balancer 2024-11-16T02:34:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, REOPEN/MOVE 2024-11-16T02:34:39,382 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, REOPEN/MOVE 2024-11-16T02:34:39,384 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=056b939bd43275b5084f24034a22ab92, regionState=CLOSING, regionLocation=667222f7cf1b,40547,1731724462323 2024-11-16T02:34:39,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] regionserver.HRegion(7855): Warmup {ENCODED => 056b939bd43275b5084f24034a22ab92, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:39,386 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, REOPEN/MOVE because future has completed 2024-11-16T02:34:39,387 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056b939bd43275b5084f24034a22ab92 columnFamilyName cf1 2024-11-16T02:34:39,387 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:39,388 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T02:34:39,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40547,1731724462323}] 2024-11-16T02:34:39,397 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/1494e7e13b0f4f7cbd7100b7879bc0d0 2024-11-16T02:34:39,404 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/3ba8bd5871ff42d2aa8c98097ddb0a24 2024-11-16T02:34:39,411 INFO [StoreFileOpener-056b939bd43275b5084f24034a22ab92-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 54eeae3876c3418ea1cbcfb1b88e72c9 2024-11-16T02:34:39,411 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/54eeae3876c3418ea1cbcfb1b88e72c9 2024-11-16T02:34:39,411 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(327): Store=056b939bd43275b5084f24034a22ab92/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:39,411 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,412 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056b939bd43275b5084f24034a22ab92 columnFamilyName cf2 2024-11-16T02:34:39,412 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:39,420 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf2/491810076f2c4278a156c746cfa3f212 2024-11-16T02:34:39,426 INFO [StoreFileOpener-056b939bd43275b5084f24034a22ab92-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d4fbca7a4801448ca5da7ba1407486c5 2024-11-16T02:34:39,426 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf2/d4fbca7a4801448ca5da7ba1407486c5 2024-11-16T02:34:39,426 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(327): Store=056b939bd43275b5084f24034a22ab92/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] regionserver.HRegion(1722): Closing 056b939bd43275b5084f24034a22ab92, disabling compactions & flushes 2024-11-16T02:34:39,426 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. after waiting 0 ms 2024-11-16T02:34:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] regionserver.HRegion(1676): Region close journal for 056b939bd43275b5084f24034a22ab92: Waiting for close lock at 1731724479426Disabling compacts and flushes for region at 1731724479426Disabling writes for close at 1731724479426Writing region close event to WAL at 1731724479427 (+1 ms)Closed at 1731724479427 2024-11-16T02:34:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-16T02:34:39,542 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,542 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T02:34:39,542 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 056b939bd43275b5084f24034a22ab92, disabling compactions & flushes 2024-11-16T02:34:39,542 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,543 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,543 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. after waiting 0 ms 2024-11-16T02:34:39,543 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,543 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/3ba8bd5871ff42d2aa8c98097ddb0a24, hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/54eeae3876c3418ea1cbcfb1b88e72c9] to archive 2024-11-16T02:34:39,546 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T02:34:39,550 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/3ba8bd5871ff42d2aa8c98097ddb0a24 to hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/3ba8bd5871ff42d2aa8c98097ddb0a24 2024-11-16T02:34:39,552 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/54eeae3876c3418ea1cbcfb1b88e72c9 to hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/54eeae3876c3418ea1cbcfb1b88e72c9 2024-11-16T02:34:39,565 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf2/d4fbca7a4801448ca5da7ba1407486c5] to archive 2024-11-16T02:34:39,566 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T02:34:39,568 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf2/d4fbca7a4801448ca5da7ba1407486c5 to hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf2/d4fbca7a4801448ca5da7ba1407486c5 2024-11-16T02:34:39,573 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-16T02:34:39,574 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,574 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 056b939bd43275b5084f24034a22ab92: Waiting for close lock at 1731724479542Running coprocessor pre-close hooks at 1731724479542Disabling compacts and flushes for region at 1731724479542Disabling writes for close at 1731724479543 (+1 ms)Writing region close event to WAL at 1731724479569 (+26 ms)Running coprocessor post-close hooks at 1731724479574 (+5 ms)Closed at 1731724479574 2024-11-16T02:34:39,574 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 056b939bd43275b5084f24034a22ab92 move to 667222f7cf1b,40209,1731724462418 record at close sequenceid=12 2024-11-16T02:34:39,578 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,579 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=056b939bd43275b5084f24034a22ab92, regionState=CLOSED 2024-11-16T02:34:39,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40547,1731724462323 because future has completed 2024-11-16T02:34:39,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-16T02:34:39,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40547,1731724462323 in 195 msec 2024-11-16T02:34:39,587 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, REOPEN/MOVE; state=CLOSED, location=667222f7cf1b,40209,1731724462418; forceNewPlan=false, retain=false 2024-11-16T02:34:39,737 INFO [667222f7cf1b:38401 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T02:34:39,738 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=056b939bd43275b5084f24034a22ab92, regionState=OPENING, regionLocation=667222f7cf1b,40209,1731724462418 2024-11-16T02:34:39,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, REOPEN/MOVE because future has completed 2024-11-16T02:34:39,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40209,1731724462418}] 2024-11-16T02:34:39,902 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,902 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 056b939bd43275b5084f24034a22ab92, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:39,903 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,903 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:39,904 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,904 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,906 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,908 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056b939bd43275b5084f24034a22ab92 columnFamilyName cf1 2024-11-16T02:34:39,909 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:39,959 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/1494e7e13b0f4f7cbd7100b7879bc0d0 2024-11-16T02:34:39,960 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(327): Store=056b939bd43275b5084f24034a22ab92/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:39,960 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,962 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056b939bd43275b5084f24034a22ab92 columnFamilyName cf2 2024-11-16T02:34:39,962 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:39,971 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf2/491810076f2c4278a156c746cfa3f212 2024-11-16T02:34:39,971 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(327): Store=056b939bd43275b5084f24034a22ab92/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:39,971 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,972 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,974 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,975 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,975 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,976 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-16T02:34:39,977 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,978 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 056b939bd43275b5084f24034a22ab92; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74507317, jitterRate=0.11024554073810577}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-16T02:34:39,978 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:39,979 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 056b939bd43275b5084f24034a22ab92: Running coprocessor pre-open hook at 1731724479904Writing region info on filesystem at 1731724479904Initializing all the Stores at 1731724479906 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724479906Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724479906Cleaning up temporary data from old regions at 1731724479975 (+69 ms)Running coprocessor post-open hooks at 1731724479978 (+3 ms)Region opened successfully at 1731724479979 (+1 ms) 2024-11-16T02:34:39,980 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., pid=12, masterSystemTime=1731724479894 2024-11-16T02:34:39,982 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,982 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:39,983 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=056b939bd43275b5084f24034a22ab92, regionState=OPEN, openSeqNum=18, regionLocation=667222f7cf1b,40209,1731724462418 2024-11-16T02:34:39,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40209,1731724462418 because future has completed 2024-11-16T02:34:39,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-16T02:34:39,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,40209,1731724462418 in 244 msec 2024-11-16T02:34:39,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, REOPEN/MOVE in 609 msec 2024-11-16T02:34:40,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T02:34:40,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T02:34:40,040 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 667222f7cf1b,40209,1731724462418: testing ***** 2024-11-16T02:34:40,040 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T02:34:40,043 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T02:34:40,045 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T02:34:40,048 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T02:34:40,049 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T02:34:40,057 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 473184368 }, "NonHeapMemoryUsage": { "committed": 171900928, "init": 7667712, "max": -1, "used": 169218664 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "667222f7cf1b", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "667222f7cf1b", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:51370", "tag.serverName": "667222f7cf1b,40547,1731724462323", "tag.clusterId": "5c137c35-6caf-4e9b-872c-afcb989f7817", "tag.Context": "regionserver", "tag.Hostname": "667222f7cf1b", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731724462323, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 3, "ScanTime_mean": 1, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 2, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 6, "Put_mean": 4, "Put_25th_percentile": 3, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 5, "Put_95th_percentile": 5, "Put_98th_percentile": 5, "Put_99th_percentile": 5, "Put_99.9th_percentile": 5, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-16T02:34:40,060 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38401 {}] master.MasterRpcServices(700): 667222f7cf1b,40209,1731724462418 reported a fatal error: ***** ABORTING region server 667222f7cf1b,40209,1731724462418: testing ***** 2024-11-16T02:34:40,062 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '667222f7cf1b,40209,1731724462418' ***** 2024-11-16T02:34:40,062 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-16T02:34:40,062 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T02:34:40,062 INFO [RS:1;667222f7cf1b:40209 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T02:34:40,062 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T02:34:40,063 INFO [RS:1;667222f7cf1b:40209 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T02:34:40,063 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(3091): Received CLOSE for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,063 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(956): aborting server 667222f7cf1b,40209,1731724462418 2024-11-16T02:34:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40547 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:59700 deadline: 1731724540063, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=667222f7cf1b port=40209 startCode=1731724462418. As of locationSeqNum=12. 2024-11-16T02:34:40,063 INFO [RS:1;667222f7cf1b:40209 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T02:34:40,063 INFO [RS:1;667222f7cf1b:40209 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;667222f7cf1b:40209. 2024-11-16T02:34:40,063 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 056b939bd43275b5084f24034a22ab92, disabling compactions & flushes 2024-11-16T02:34:40,063 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:40,063 DEBUG [RS:1;667222f7cf1b:40209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T02:34:40,063 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:40,063 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. after waiting 0 ms 2024-11-16T02:34:40,063 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:40,064 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40547,1731724462323, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40547,1731724462323, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=667222f7cf1b port=40209 startCode=1731724462418. As of locationSeqNum=12. 2024-11-16T02:34:40,064 DEBUG [RS:1;667222f7cf1b:40209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T02:34:40,064 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40547,1731724462323, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=667222f7cf1b port=40209 startCode=1731724462418. As of locationSeqNum=12. 2024-11-16T02:34:40,064 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40547,1731724462323, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=667222f7cf1b port=40209 startCode=1731724462418. As of locationSeqNum=12. 2024-11-16T02:34:40,064 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T02:34:40,064 DEBUG [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(1325): Online Regions={056b939bd43275b5084f24034a22ab92=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.} 2024-11-16T02:34:40,065 DEBUG [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(1351): Waiting on 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,066 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:40,066 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 056b939bd43275b5084f24034a22ab92: Waiting for close lock at 1731724480063Running coprocessor pre-close hooks at 1731724480063Disabling compacts and flushes for region at 1731724480063Disabling writes for close at 1731724480063Writing region close event to WAL at 1731724480066 (+3 ms)Running coprocessor post-close hooks at 1731724480066Closed at 1731724480066 2024-11-16T02:34:40,066 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:40,117 INFO [regionserver/667222f7cf1b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T02:34:40,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 667222f7cf1b,40209,1731724462418 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:40,177 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 667222f7cf1b,40209,1731724462418 aborting 2024-11-16T02:34:40,177 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 667222f7cf1b,40209,1731724462418 aborting 2024-11-16T02:34:40,177 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=12 from cache 2024-11-16T02:34:40,265 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(976): stopping server 667222f7cf1b,40209,1731724462418; all regions closed. 2024-11-16T02:34:40,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741834_1010 (size=2288) 2024-11-16T02:34:40,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741834_1010 (size=2288) 2024-11-16T02:34:40,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741834_1010 (size=2288) 2024-11-16T02:34:40,270 DEBUG [RS:1;667222f7cf1b:40209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T02:34:40,270 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T02:34:40,270 INFO [RS:1;667222f7cf1b:40209 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T02:34:40,270 INFO [RS:1;667222f7cf1b:40209 {}] hbase.ChoreService(370): Chore service for: regionserver/667222f7cf1b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T02:34:40,271 INFO [regionserver/667222f7cf1b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T02:34:40,271 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T02:34:40,271 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T02:34:40,271 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T02:34:40,271 INFO [RS:1;667222f7cf1b:40209 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T02:34:40,271 INFO [RS:1;667222f7cf1b:40209 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40209 2024-11-16T02:34:40,361 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/667222f7cf1b,40209,1731724462418 2024-11-16T02:34:40,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T02:34:40,361 INFO [RS:1;667222f7cf1b:40209 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T02:34:40,363 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [667222f7cf1b,40209,1731724462418] 2024-11-16T02:34:40,389 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=18] 2024-11-16T02:34:40,389 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/667222f7cf1b,40209,1731724462418 already deleted, retry=false 2024-11-16T02:34:40,390 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 667222f7cf1b,40209,1731724462418 on 667222f7cf1b,38401,1731724461545 2024-11-16T02:34:40,390 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 667222f7cf1b:40209 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 667222f7cf1b/172.17.0.3:40209 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:40,391 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=18, error=java.net.ConnectException: Call to address=667222f7cf1b:40209 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 667222f7cf1b/172.17.0.3:40209 2024-11-16T02:34:40,391 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-16T02:34:40,391 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=18 from cache 2024-11-16T02:34:40,392 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 667222f7cf1b:40209 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 667222f7cf1b/172.17.0.3:40209 2024-11-16T02:34:40,395 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 667222f7cf1b,40209,1731724462418, splitWal=true, meta=false 2024-11-16T02:34:40,398 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 667222f7cf1b,40209,1731724462418, splitWal=true, meta=false 2024-11-16T02:34:40,398 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for 667222f7cf1b,40209,1731724462418 (carryingMeta=false) 667222f7cf1b,40209,1731724462418/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@779ca06[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-16T02:34:40,400 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 667222f7cf1b,40209,1731724462418 had 1 regions 2024-11-16T02:34:40,401 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 667222f7cf1b,40209,1731724462418, splitWal=true, meta=false, isMeta: false 2024-11-16T02:34:40,402 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting 2024-11-16T02:34:40,404 INFO [PEWorker-1 {}] master.SplitWALManager(105): 667222f7cf1b,40209,1731724462418 WAL count=1, meta=false 2024-11-16T02:34:40,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 667222f7cf1b%2C40209%2C1731724462418.1731724464057}] 2024-11-16T02:34:40,412 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=667222f7cf1b,39421,1731724462467 2024-11-16T02:34:40,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 667222f7cf1b%2C40209%2C1731724462418.1731724464057, worker=667222f7cf1b,39421,1731724462467}] 2024-11-16T02:34:40,480 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T02:34:40,480 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40209-0x101417e3e890002, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T02:34:40,480 INFO [RS:1;667222f7cf1b:40209 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T02:34:40,480 INFO [RS:1;667222f7cf1b:40209 {}] regionserver.HRegionServer(1031): Exiting; stopping=667222f7cf1b,40209,1731724462418; zookeeper connection closed. 2024-11-16T02:34:40,480 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7f5a1920 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7f5a1920 2024-11-16T02:34:40,575 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T02:34:40,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44427, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T02:34:40,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-16T02:34:40,596 INFO [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting/667222f7cf1b%2C40209%2C1731724462418.1731724464057, size=2.2 K (2288bytes) 2024-11-16T02:34:40,597 INFO [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting/667222f7cf1b%2C40209%2C1731724462418.1731724464057 2024-11-16T02:34:40,597 INFO [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting/667222f7cf1b%2C40209%2C1731724462418.1731724464057 after 0ms 2024-11-16T02:34:40,622 DEBUG [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:40,622 INFO [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting/667222f7cf1b%2C40209%2C1731724462418.1731724464057 took 26ms 2024-11-16T02:34:40,629 DEBUG [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 056b939bd43275b5084f24034a22ab92: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-16T02:34:40,629 DEBUG [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting/667222f7cf1b%2C40209%2C1731724462418.1731724464057 so closing down 2024-11-16T02:34:40,629 DEBUG [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:40,629 INFO [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:40,629 INFO [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting/667222f7cf1b%2C40209%2C1731724462418.1731724464057, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-16T02:34:40,629 DEBUG [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting/667222f7cf1b%2C40209%2C1731724462418.1731724464057, journal: Splitting hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting/667222f7cf1b%2C40209%2C1731724462418.1731724464057, size=2.2 K (2288bytes) at 1731724480597Finishing writing output for hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting/667222f7cf1b%2C40209%2C1731724462418.1731724464057 so closing down at 1731724480629 (+32 ms)3 split writer threads finished at 1731724480629Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting/667222f7cf1b%2C40209%2C1731724462418.1731724464057, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1731724480629 2024-11-16T02:34:40,629 DEBUG [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting/667222f7cf1b%2C40209%2C1731724462418.1731724464057 2024-11-16T02:34:40,630 DEBUG [RS_LOG_REPLAY_OPS-regionserver/667222f7cf1b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-16T02:34:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38401 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-16T02:34:40,636 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting/667222f7cf1b%2C40209%2C1731724462418.1731724464057 to hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/oldWALs 2024-11-16T02:34:40,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-16T02:34:40,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 667222f7cf1b%2C40209%2C1731724462418.1731724464057, worker=667222f7cf1b,39421,1731724462467 in 222 msec 2024-11-16T02:34:40,640 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=667222f7cf1b,39421,1731724462467 2024-11-16T02:34:40,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-16T02:34:40,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 667222f7cf1b%2C40209%2C1731724462418.1731724464057, worker=667222f7cf1b,39421,1731724462467 in 235 msec 2024-11-16T02:34:40,645 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting dir is empty, no logs to split. 2024-11-16T02:34:40,645 INFO [PEWorker-1 {}] master.SplitWALManager(105): 667222f7cf1b,40209,1731724462418 WAL count=0, meta=false 2024-11-16T02:34:40,645 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 667222f7cf1b,40209,1731724462418 WAL splitting is done? wals=0, meta=false 2024-11-16T02:34:40,647 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 667222f7cf1b,40209,1731724462418 failed, ignore...File hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/WALs/667222f7cf1b,40209,1731724462418-splitting does not exist. 2024-11-16T02:34:40,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, ASSIGN}] 2024-11-16T02:34:40,651 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, ASSIGN 2024-11-16T02:34:40,652 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T02:34:40,699 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=18] 2024-11-16T02:34:40,699 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 667222f7cf1b:40209 this server is in the failed servers list 2024-11-16T02:34:40,699 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=667222f7cf1b:40209 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 667222f7cf1b:40209 2024-11-16T02:34:40,700 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 667222f7cf1b:40209 2024-11-16T02:34:40,700 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,40209,1731724462418, seqNum=18 from cache 2024-11-16T02:34:40,802 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(204): Hosts are {667222f7cf1b=0} racks are {/default-rack=0} 2024-11-16T02:34:40,802 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T02:34:40,802 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T02:34:40,802 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T02:34:40,802 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T02:34:40,802 INFO [667222f7cf1b:38401 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T02:34:40,802 INFO [667222f7cf1b:38401 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T02:34:40,802 DEBUG [667222f7cf1b:38401 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T02:34:40,803 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=056b939bd43275b5084f24034a22ab92, regionState=OPENING, regionLocation=667222f7cf1b,39421,1731724462467 2024-11-16T02:34:40,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, ASSIGN because future has completed 2024-11-16T02:34:40,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,39421,1731724462467}] 2024-11-16T02:34:40,969 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:40,970 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 056b939bd43275b5084f24034a22ab92, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:40,971 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,971 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:40,971 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,972 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,974 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,975 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056b939bd43275b5084f24034a22ab92 columnFamilyName cf1 2024-11-16T02:34:40,975 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:40,983 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf1/1494e7e13b0f4f7cbd7100b7879bc0d0 2024-11-16T02:34:40,983 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(327): Store=056b939bd43275b5084f24034a22ab92/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:40,983 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,984 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056b939bd43275b5084f24034a22ab92 columnFamilyName cf2 2024-11-16T02:34:40,984 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:40,991 DEBUG [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/cf2/491810076f2c4278a156c746cfa3f212 2024-11-16T02:34:40,992 INFO [StoreOpener-056b939bd43275b5084f24034a22ab92-1 {}] regionserver.HStore(327): Store=056b939bd43275b5084f24034a22ab92/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:40,992 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,993 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,994 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,995 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,995 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,995 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-16T02:34:40,996 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,997 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 056b939bd43275b5084f24034a22ab92; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69445604, jitterRate=0.03482013940811157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-16T02:34:40,997 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:34:40,998 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 056b939bd43275b5084f24034a22ab92: Running coprocessor pre-open hook at 1731724480972Writing region info on filesystem at 1731724480972Initializing all the Stores at 1731724480973 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724480973Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724480974 (+1 ms)Cleaning up temporary data from old regions at 1731724480995 (+21 ms)Running coprocessor post-open hooks at 1731724480998 (+3 ms)Region opened successfully at 1731724480998 2024-11-16T02:34:40,999 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., pid=17, masterSystemTime=1731724480959 2024-11-16T02:34:41,002 DEBUG [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:41,002 INFO [RS_OPEN_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:34:41,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=056b939bd43275b5084f24034a22ab92, regionState=OPEN, openSeqNum=18, regionLocation=667222f7cf1b,39421,1731724462467 2024-11-16T02:34:41,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,39421,1731724462467 because future has completed 2024-11-16T02:34:41,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-16T02:34:41,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 056b939bd43275b5084f24034a22ab92, server=667222f7cf1b,39421,1731724462467 in 200 msec 2024-11-16T02:34:41,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-16T02:34:41,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=056b939bd43275b5084f24034a22ab92, ASSIGN in 360 msec 2024-11-16T02:34:41,012 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 667222f7cf1b,40209,1731724462418 after splitting done 2024-11-16T02:34:41,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 667222f7cf1b,40209,1731724462418, splitWal=true, meta=false in 621 msec 2024-11-16T02:34:41,210 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92., hostname=667222f7cf1b,39421,1731724462467, seqNum=18] 2024-11-16T02:34:41,211 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T02:34:41,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T02:34:41,228 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=407 (was 411), OpenFileDescriptor=1050 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 384), ProcessCount=11 (was 11), AvailableMemoryMB=4139 (was 4202) 2024-11-16T02:34:41,228 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-16T02:34:41,241 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=407, OpenFileDescriptor=1050, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=4139 2024-11-16T02:34:41,241 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-16T02:34:41,258 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:41,260 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:41,261 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:41,263 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-92947842, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/hregion-92947842, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:41,274 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-92947842/hregion-92947842.1731724481263, exclude list is [], retry=0 2024-11-16T02:34:41,277 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:41,277 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:41,277 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:41,302 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-92947842/hregion-92947842.1731724481263 2024-11-16T02:34:41,303 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:41,303 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e6af9d24429413e1f2745c0487ced38e, NAME => 'testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43879/hbase 2024-11-16T02:34:41,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741893_1071 (size=67) 2024-11-16T02:34:41,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741893_1071 (size=67) 2024-11-16T02:34:41,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741893_1071 (size=67) 2024-11-16T02:34:41,312 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:41,314 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,315 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6af9d24429413e1f2745c0487ced38e columnFamilyName a 2024-11-16T02:34:41,315 DEBUG [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:41,315 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(327): Store=e6af9d24429413e1f2745c0487ced38e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:41,316 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,317 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6af9d24429413e1f2745c0487ced38e columnFamilyName b 2024-11-16T02:34:41,317 DEBUG [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:41,317 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(327): Store=e6af9d24429413e1f2745c0487ced38e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:41,318 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,319 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6af9d24429413e1f2745c0487ced38e columnFamilyName c 2024-11-16T02:34:41,319 DEBUG [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:41,319 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(327): Store=e6af9d24429413e1f2745c0487ced38e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:41,319 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,320 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,321 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,322 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,322 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,322 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:41,324 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,326 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:41,327 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e6af9d24429413e1f2745c0487ced38e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73678271, jitterRate=0.09789179265499115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:41,327 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e6af9d24429413e1f2745c0487ced38e: Writing region info on filesystem at 1731724481312Initializing all the Stores at 1731724481313 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724481313Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724481313Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724481314 (+1 ms)Cleaning up temporary data from old regions at 1731724481322 (+8 ms)Region opened successfully at 1731724481327 (+5 ms) 2024-11-16T02:34:41,327 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e6af9d24429413e1f2745c0487ced38e, disabling compactions & flushes 2024-11-16T02:34:41,328 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e. 2024-11-16T02:34:41,328 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e. 2024-11-16T02:34:41,328 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e. after waiting 0 ms 2024-11-16T02:34:41,328 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e. 2024-11-16T02:34:41,328 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e. 2024-11-16T02:34:41,328 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e6af9d24429413e1f2745c0487ced38e: Waiting for close lock at 1731724481327Disabling compacts and flushes for region at 1731724481327Disabling writes for close at 1731724481328 (+1 ms)Writing region close event to WAL at 1731724481328Closed at 1731724481328 2024-11-16T02:34:41,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741892_1070 (size=175) 2024-11-16T02:34:41,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741892_1070 (size=175) 2024-11-16T02:34:41,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741892_1070 (size=175) 2024-11-16T02:34:41,333 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:41,333 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-92947842:(num 1731724481263) 2024-11-16T02:34:41,333 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:41,335 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:41,348 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336, exclude list is [], retry=0 2024-11-16T02:34:41,350 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:41,350 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:41,351 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:41,375 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336 2024-11-16T02:34:41,376 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:41,376 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e6af9d24429413e1f2745c0487ced38e, NAME => 'testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:41,376 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:41,376 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,376 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,378 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,379 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6af9d24429413e1f2745c0487ced38e columnFamilyName a 2024-11-16T02:34:41,379 DEBUG [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:41,380 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(327): Store=e6af9d24429413e1f2745c0487ced38e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:41,380 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,381 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6af9d24429413e1f2745c0487ced38e columnFamilyName b 2024-11-16T02:34:41,381 DEBUG [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:41,381 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(327): Store=e6af9d24429413e1f2745c0487ced38e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:41,381 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,382 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6af9d24429413e1f2745c0487ced38e columnFamilyName c 2024-11-16T02:34:41,382 DEBUG [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:41,383 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(327): Store=e6af9d24429413e1f2745c0487ced38e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:41,383 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,384 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,385 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,386 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,386 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,386 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:41,388 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,388 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e6af9d24429413e1f2745c0487ced38e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61173158, jitterRate=-0.0884489119052887}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:41,389 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e6af9d24429413e1f2745c0487ced38e: Writing region info on filesystem at 1731724481376Initializing all the Stores at 1731724481378 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724481378Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724481378Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724481378Cleaning up temporary data from old regions at 1731724481386 (+8 ms)Region opened successfully at 1731724481389 (+3 ms) 2024-11-16T02:34:41,419 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e6af9d24429413e1f2745c0487ced38e 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-16T02:34:41,440 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/.tmp/a/8b68528c44d74873b70b05489d4605be is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731724481389/Put/seqid=0 2024-11-16T02:34:41,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741895_1073 (size=5958) 2024-11-16T02:34:41,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741895_1073 (size=5958) 2024-11-16T02:34:41,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741895_1073 (size=5958) 2024-11-16T02:34:41,451 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/.tmp/a/8b68528c44d74873b70b05489d4605be 2024-11-16T02:34:41,477 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/.tmp/b/525fa79e873f48c6ab3b2c885627f84d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731724481398/Put/seqid=0 2024-11-16T02:34:41,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741896_1074 (size=5958) 2024-11-16T02:34:41,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741896_1074 (size=5958) 2024-11-16T02:34:41,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741896_1074 (size=5958) 2024-11-16T02:34:41,485 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/.tmp/b/525fa79e873f48c6ab3b2c885627f84d 2024-11-16T02:34:41,514 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/.tmp/c/39a562fe8efe4d068c649eed547f02bf is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731724481407/Put/seqid=0 2024-11-16T02:34:41,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741897_1075 (size=5958) 2024-11-16T02:34:41,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741897_1075 (size=5958) 2024-11-16T02:34:41,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741897_1075 (size=5958) 2024-11-16T02:34:41,527 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/.tmp/c/39a562fe8efe4d068c649eed547f02bf 2024-11-16T02:34:41,533 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/.tmp/a/8b68528c44d74873b70b05489d4605be as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/a/8b68528c44d74873b70b05489d4605be 2024-11-16T02:34:41,541 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/a/8b68528c44d74873b70b05489d4605be, entries=10, sequenceid=33, filesize=5.8 K 2024-11-16T02:34:41,542 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/.tmp/b/525fa79e873f48c6ab3b2c885627f84d as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/b/525fa79e873f48c6ab3b2c885627f84d 2024-11-16T02:34:41,549 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/b/525fa79e873f48c6ab3b2c885627f84d, entries=10, sequenceid=33, filesize=5.8 K 2024-11-16T02:34:41,551 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/.tmp/c/39a562fe8efe4d068c649eed547f02bf as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/c/39a562fe8efe4d068c649eed547f02bf 2024-11-16T02:34:41,558 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/c/39a562fe8efe4d068c649eed547f02bf, entries=10, sequenceid=33, filesize=5.8 K 2024-11-16T02:34:41,560 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for e6af9d24429413e1f2745c0487ced38e in 142ms, sequenceid=33, compaction requested=false 2024-11-16T02:34:41,560 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e6af9d24429413e1f2745c0487ced38e: 2024-11-16T02:34:41,560 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e6af9d24429413e1f2745c0487ced38e, disabling compactions & flushes 2024-11-16T02:34:41,560 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e. 2024-11-16T02:34:41,560 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e. 2024-11-16T02:34:41,560 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e. after waiting 0 ms 2024-11-16T02:34:41,560 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e. 2024-11-16T02:34:41,564 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e. 2024-11-16T02:34:41,564 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e6af9d24429413e1f2745c0487ced38e: Waiting for close lock at 1731724481560Disabling compacts and flushes for region at 1731724481560Disabling writes for close at 1731724481560Writing region close event to WAL at 1731724481564 (+4 ms)Closed at 1731724481564 2024-11-16T02:34:41,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741894_1072 (size=6311) 2024-11-16T02:34:41,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741894_1072 (size=6311) 2024-11-16T02:34:41,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741894_1072 (size=6311) 2024-11-16T02:34:41,573 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/b/525fa79e873f48c6ab3b2c885627f84d to hdfs://localhost:43879/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/b/525fa79e873f48c6ab3b2c885627f84d 2024-11-16T02:34:41,588 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336, size=6.2 K (6311bytes) 2024-11-16T02:34:41,588 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336 2024-11-16T02:34:41,589 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336 after 1ms 2024-11-16T02:34:41,617 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:41,617 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336 took 29ms 2024-11-16T02:34:41,620 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336 so closing down 2024-11-16T02:34:41,620 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:41,621 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731724481336.temp 2024-11-16T02:34:41,643 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/0000000000000000003-wal.1731724481336.temp 2024-11-16T02:34:41,644 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:41,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741898_1076 (size=5575) 2024-11-16T02:34:41,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741898_1076 (size=5575) 2024-11-16T02:34:41,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741898_1076 (size=5575) 2024-11-16T02:34:41,650 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/0000000000000000003-wal.1731724481336.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-16T02:34:41,651 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/0000000000000000003-wal.1731724481336.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/0000000000000000032 2024-11-16T02:34:41,651 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 33 ms; skipped=2; WAL=hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-16T02:34:41,651 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336, journal: Splitting hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336, size=6.2 K (6311bytes) at 1731724481588Finishing writing output for hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336 so closing down at 1731724481620 (+32 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/0000000000000000003-wal.1731724481336.temp at 1731724481643 (+23 ms)3 split writer threads finished at 1731724481644 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/0000000000000000003-wal.1731724481336.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731724481650 (+6 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/0000000000000000003-wal.1731724481336.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/0000000000000000032 at 1731724481651 (+1 ms)Processed 32 edits across 1 Regions in 33 ms; skipped=2; WAL=hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1731724481651 2024-11-16T02:34:41,652 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481336 to hdfs://localhost:43879/hbase/oldWALs/wal.1731724481336 2024-11-16T02:34:41,653 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/0000000000000000032 2024-11-16T02:34:41,653 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:41,655 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:41,668 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481655, exclude list is [], retry=0 2024-11-16T02:34:41,671 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:41,671 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:41,671 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:41,695 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481655 2024-11-16T02:34:41,695 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:41,695 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e6af9d24429413e1f2745c0487ced38e, NAME => 'testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:41,695 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:41,695 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,695 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,697 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,698 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6af9d24429413e1f2745c0487ced38e columnFamilyName a 2024-11-16T02:34:41,698 DEBUG [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:41,703 DEBUG [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/a/8b68528c44d74873b70b05489d4605be 2024-11-16T02:34:41,703 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(327): Store=e6af9d24429413e1f2745c0487ced38e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:41,703 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,705 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6af9d24429413e1f2745c0487ced38e columnFamilyName b 2024-11-16T02:34:41,705 DEBUG [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:41,705 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(327): Store=e6af9d24429413e1f2745c0487ced38e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:41,706 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,706 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6af9d24429413e1f2745c0487ced38e columnFamilyName c 2024-11-16T02:34:41,706 DEBUG [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:41,712 DEBUG [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/c/39a562fe8efe4d068c649eed547f02bf 2024-11-16T02:34:41,712 INFO [StoreOpener-e6af9d24429413e1f2745c0487ced38e-1 {}] regionserver.HStore(327): Store=e6af9d24429413e1f2745c0487ced38e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:41,712 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,713 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,715 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,716 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/0000000000000000032 2024-11-16T02:34:41,741 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:41,743 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/0000000000000000032 2024-11-16T02:34:41,743 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e6af9d24429413e1f2745c0487ced38e 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-16T02:34:41,758 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/.tmp/b/a40f8374cd874068bf949d1e8b081e9d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731724481398/Put/seqid=0 2024-11-16T02:34:41,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741900_1078 (size=5958) 2024-11-16T02:34:41,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741900_1078 (size=5958) 2024-11-16T02:34:41,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741900_1078 (size=5958) 2024-11-16T02:34:41,765 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/.tmp/b/a40f8374cd874068bf949d1e8b081e9d 2024-11-16T02:34:41,771 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/.tmp/b/a40f8374cd874068bf949d1e8b081e9d as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/b/a40f8374cd874068bf949d1e8b081e9d 2024-11-16T02:34:41,776 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/b/a40f8374cd874068bf949d1e8b081e9d, entries=10, sequenceid=32, filesize=5.8 K 2024-11-16T02:34:41,777 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for e6af9d24429413e1f2745c0487ced38e in 33ms, sequenceid=32, compaction requested=false; wal=null 2024-11-16T02:34:41,777 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/0000000000000000032 2024-11-16T02:34:41,778 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,778 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,779 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:41,780 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e6af9d24429413e1f2745c0487ced38e 2024-11-16T02:34:41,782 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/e6af9d24429413e1f2745c0487ced38e/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-16T02:34:41,783 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e6af9d24429413e1f2745c0487ced38e; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63196656, jitterRate=-0.05829644203186035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:41,784 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e6af9d24429413e1f2745c0487ced38e: Writing region info on filesystem at 1731724481695Initializing all the Stores at 1731724481696 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724481696Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724481697 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724481697Obtaining lock to block concurrent updates at 1731724481743 (+46 ms)Preparing flush snapshotting stores in e6af9d24429413e1f2745c0487ced38e at 1731724481743Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731724481743Flushing stores of testReplayEditsWrittenViaHRegion,,1731724481259.e6af9d24429413e1f2745c0487ced38e. at 1731724481743Flushing e6af9d24429413e1f2745c0487ced38e/b: creating writer at 1731724481743Flushing e6af9d24429413e1f2745c0487ced38e/b: appending metadata at 1731724481757 (+14 ms)Flushing e6af9d24429413e1f2745c0487ced38e/b: closing flushed file at 1731724481757Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b523ee9: reopening flushed file at 1731724481770 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for e6af9d24429413e1f2745c0487ced38e in 33ms, sequenceid=32, compaction requested=false; wal=null at 1731724481777 (+7 ms)Cleaning up temporary data from old regions at 1731724481778 (+1 ms)Region opened successfully at 1731724481784 (+6 ms) 2024-11-16T02:34:41,804 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=414 (was 407) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:41580 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:34606 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:59476 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:41438 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:59284 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:34760 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1050) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 384), ProcessCount=11 (was 11), AvailableMemoryMB=4133 (was 4139) 2024-11-16T02:34:41,804 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-16T02:34:41,816 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=414, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=4132 2024-11-16T02:34:41,816 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-16T02:34:41,831 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:41,833 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:41,834 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:41,836 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57631999, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/hregion-57631999, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:41,854 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57631999/hregion-57631999.1731724481837, exclude list is [], retry=0 2024-11-16T02:34:41,857 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:41,857 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:41,858 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:41,891 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57631999/hregion-57631999.1731724481837 2024-11-16T02:34:41,892 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:41131:41131)] 2024-11-16T02:34:41,892 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 28a84f7f3906829018638c93ecbbe757, NAME => 'testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43879/hbase 2024-11-16T02:34:41,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741902_1080 (size=68) 2024-11-16T02:34:41,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741902_1080 (size=68) 2024-11-16T02:34:41,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741902_1080 (size=68) 2024-11-16T02:34:41,901 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:41,902 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:41,903 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a84f7f3906829018638c93ecbbe757 columnFamilyName a 2024-11-16T02:34:41,904 DEBUG [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:41,904 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(327): Store=28a84f7f3906829018638c93ecbbe757/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:41,904 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:41,905 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a84f7f3906829018638c93ecbbe757 columnFamilyName b 2024-11-16T02:34:41,906 DEBUG [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:41,906 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(327): Store=28a84f7f3906829018638c93ecbbe757/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:41,906 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:41,907 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a84f7f3906829018638c93ecbbe757 columnFamilyName c 2024-11-16T02:34:41,907 DEBUG [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:41,908 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(327): Store=28a84f7f3906829018638c93ecbbe757/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:41,908 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:41,909 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:41,909 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:41,910 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:41,910 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:41,911 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:41,912 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:41,914 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:41,914 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28a84f7f3906829018638c93ecbbe757; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70403094, jitterRate=0.049087852239608765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:41,915 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28a84f7f3906829018638c93ecbbe757: Writing region info on filesystem at 1731724481901Initializing all the Stores at 1731724481901Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724481901Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724481902 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724481902Cleaning up temporary data from old regions at 1731724481910 (+8 ms)Region opened successfully at 1731724481915 (+5 ms) 2024-11-16T02:34:41,915 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28a84f7f3906829018638c93ecbbe757, disabling compactions & flushes 2024-11-16T02:34:41,915 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. 2024-11-16T02:34:41,915 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. 2024-11-16T02:34:41,915 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. after waiting 0 ms 2024-11-16T02:34:41,915 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. 2024-11-16T02:34:41,916 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. 2024-11-16T02:34:41,916 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28a84f7f3906829018638c93ecbbe757: Waiting for close lock at 1731724481915Disabling compacts and flushes for region at 1731724481915Disabling writes for close at 1731724481915Writing region close event to WAL at 1731724481916 (+1 ms)Closed at 1731724481916 2024-11-16T02:34:41,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741901_1079 (size=175) 2024-11-16T02:34:41,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741901_1079 (size=175) 2024-11-16T02:34:41,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741901_1079 (size=175) 2024-11-16T02:34:41,921 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:41,921 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57631999:(num 1731724481837) 2024-11-16T02:34:41,921 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:41,923 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:41,929 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-16T02:34:41,929 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-16T02:34:41,930 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-16T02:34:41,930 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-16T02:34:41,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-16T02:34:41,932 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-16T02:34:41,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-16T02:34:41,933 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-16T02:34:41,937 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923, exclude list is [], retry=0 2024-11-16T02:34:41,940 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:41,940 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:41,940 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:41,964 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923 2024-11-16T02:34:41,964 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477)] 2024-11-16T02:34:42,036 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 28a84f7f3906829018638c93ecbbe757, NAME => 'testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:42,039 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,039 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:42,039 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,039 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,041 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,042 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a84f7f3906829018638c93ecbbe757 columnFamilyName a 2024-11-16T02:34:42,042 DEBUG [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:42,043 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(327): Store=28a84f7f3906829018638c93ecbbe757/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:42,043 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,044 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a84f7f3906829018638c93ecbbe757 columnFamilyName b 2024-11-16T02:34:42,044 DEBUG [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:42,044 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(327): Store=28a84f7f3906829018638c93ecbbe757/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:42,044 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,045 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a84f7f3906829018638c93ecbbe757 columnFamilyName c 2024-11-16T02:34:42,045 DEBUG [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:42,046 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(327): Store=28a84f7f3906829018638c93ecbbe757/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:42,046 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,047 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,048 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,049 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,049 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,050 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:42,051 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,052 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28a84f7f3906829018638c93ecbbe757; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67318050, jitterRate=0.003117114305496216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:42,052 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:42,052 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28a84f7f3906829018638c93ecbbe757: Running coprocessor pre-open hook at 1731724482040Writing region info on filesystem at 1731724482040Initializing all the Stores at 1731724482041 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724482041Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724482041Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724482041Cleaning up temporary data from old regions at 1731724482049 (+8 ms)Running coprocessor post-open hooks at 1731724482052 (+3 ms)Region opened successfully at 1731724482052 2024-11-16T02:34:42,063 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28a84f7f3906829018638c93ecbbe757 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-16T02:34:42,065 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28a84f7f3906829018638c93ecbbe757/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:43,065 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28a84f7f3906829018638c93ecbbe757/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:43,685 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T02:34:44,067 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28a84f7f3906829018638c93ecbbe757/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:45,069 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28a84f7f3906829018638c93ecbbe757/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:46,069 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28a84f7f3906829018638c93ecbbe757/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:47,070 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28a84f7f3906829018638c93ecbbe757/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:48,071 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28a84f7f3906829018638c93ecbbe757/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:49,072 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28a84f7f3906829018638c93ecbbe757/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:50,074 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28a84f7f3906829018638c93ecbbe757/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:50,711 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T02:34:51,075 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28a84f7f3906829018638c93ecbbe757/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:51,077 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 28a84f7f3906829018638c93ecbbe757: 2024-11-16T02:34:51,077 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. 2024-11-16T02:34:51,097 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 28a84f7f3906829018638c93ecbbe757: 2024-11-16T02:34:51,097 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-16T02:34:51,097 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28a84f7f3906829018638c93ecbbe757, disabling compactions & flushes 2024-11-16T02:34:51,097 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. 2024-11-16T02:34:51,097 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. 2024-11-16T02:34:51,097 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. after waiting 0 ms 2024-11-16T02:34:51,097 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. 2024-11-16T02:34:51,098 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. 2024-11-16T02:34:51,098 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. 2024-11-16T02:34:51,098 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28a84f7f3906829018638c93ecbbe757: Waiting for close lock at 1731724491097Running coprocessor pre-close hooks at 1731724491097Disabling compacts and flushes for region at 1731724491097Disabling writes for close at 1731724491097Writing region close event to WAL at 1731724491098 (+1 ms)Running coprocessor post-close hooks at 1731724491098Closed at 1731724491098 2024-11-16T02:34:51,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741903_1081 (size=4236) 2024-11-16T02:34:51,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741903_1081 (size=4236) 2024-11-16T02:34:51,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741903_1081 (size=4236) 2024-11-16T02:34:51,115 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923, size=4.1 K (4236bytes) 2024-11-16T02:34:51,115 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923 2024-11-16T02:34:51,116 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923 after 0ms 2024-11-16T02:34:51,140 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:51,141 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923 took 26ms 2024-11-16T02:34:51,143 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923 so closing down 2024-11-16T02:34:51,143 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:51,144 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731724481923.temp 2024-11-16T02:34:51,168 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/0000000000000000004-wal.1731724481923.temp 2024-11-16T02:34:51,168 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:51,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741904_1082 (size=3245) 2024-11-16T02:34:51,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741904_1082 (size=3245) 2024-11-16T02:34:51,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741904_1082 (size=3245) 2024-11-16T02:34:51,175 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/0000000000000000004-wal.1731724481923.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-16T02:34:51,177 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/0000000000000000004-wal.1731724481923.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/0000000000000000026 2024-11-16T02:34:51,177 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-16T02:34:51,177 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923, journal: Splitting hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923, size=4.1 K (4236bytes) at 1731724491115Finishing writing output for hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923 so closing down at 1731724491143 (+28 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/0000000000000000004-wal.1731724481923.temp at 1731724491168 (+25 ms)3 split writer threads finished at 1731724491168Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/0000000000000000004-wal.1731724481923.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731724491176 (+8 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/0000000000000000004-wal.1731724481923.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/0000000000000000026 at 1731724491177 (+1 ms)Processed 23 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1731724491177 2024-11-16T02:34:51,178 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724481923 to hdfs://localhost:43879/hbase/oldWALs/wal.1731724481923 2024-11-16T02:34:51,179 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/0000000000000000026 2024-11-16T02:34:51,179 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:51,181 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:51,193 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724491182, exclude list is [], retry=0 2024-11-16T02:34:51,196 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:51,196 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:51,196 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:51,220 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724491182 2024-11-16T02:34:51,221 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:51,221 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 28a84f7f3906829018638c93ecbbe757, NAME => 'testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:51,221 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,221 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:51,222 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,222 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,223 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,224 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a84f7f3906829018638c93ecbbe757 columnFamilyName a 2024-11-16T02:34:51,224 DEBUG [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:51,225 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(327): Store=28a84f7f3906829018638c93ecbbe757/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:51,225 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,226 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a84f7f3906829018638c93ecbbe757 columnFamilyName b 2024-11-16T02:34:51,226 DEBUG [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:51,226 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(327): Store=28a84f7f3906829018638c93ecbbe757/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:51,227 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,227 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a84f7f3906829018638c93ecbbe757 columnFamilyName c 2024-11-16T02:34:51,227 DEBUG [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:51,228 INFO [StoreOpener-28a84f7f3906829018638c93ecbbe757-1 {}] regionserver.HStore(327): Store=28a84f7f3906829018638c93ecbbe757/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:51,228 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,229 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,230 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,231 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/0000000000000000026 2024-11-16T02:34:51,256 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:51,258 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/0000000000000000026 2024-11-16T02:34:51,258 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28a84f7f3906829018638c93ecbbe757 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-16T02:34:51,278 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/.tmp/a/c1a1f5b5c2d444f08438179e922f8663 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731724491086/Put/seqid=0 2024-11-16T02:34:51,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741906_1084 (size=5523) 2024-11-16T02:34:51,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741906_1084 (size=5523) 2024-11-16T02:34:51,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741906_1084 (size=5523) 2024-11-16T02:34:51,285 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/.tmp/a/c1a1f5b5c2d444f08438179e922f8663 2024-11-16T02:34:51,304 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/.tmp/b/9267f083c233434f88616c2d411486ed is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731724491078/Put/seqid=0 2024-11-16T02:34:51,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741907_1085 (size=5524) 2024-11-16T02:34:51,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741907_1085 (size=5524) 2024-11-16T02:34:51,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741907_1085 (size=5524) 2024-11-16T02:34:51,311 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/.tmp/b/9267f083c233434f88616c2d411486ed 2024-11-16T02:34:51,330 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/.tmp/c/e1a56f58ba1f4119b9124158fbb52017 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731724491082/Put/seqid=0 2024-11-16T02:34:51,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741908_1086 (size=5457) 2024-11-16T02:34:51,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741908_1086 (size=5457) 2024-11-16T02:34:51,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741908_1086 (size=5457) 2024-11-16T02:34:51,337 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/.tmp/c/e1a56f58ba1f4119b9124158fbb52017 2024-11-16T02:34:51,344 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/.tmp/a/c1a1f5b5c2d444f08438179e922f8663 as hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/a/c1a1f5b5c2d444f08438179e922f8663 2024-11-16T02:34:51,349 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/a/c1a1f5b5c2d444f08438179e922f8663, entries=7, sequenceid=26, filesize=5.4 K 2024-11-16T02:34:51,350 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/.tmp/b/9267f083c233434f88616c2d411486ed as hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/b/9267f083c233434f88616c2d411486ed 2024-11-16T02:34:51,355 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/b/9267f083c233434f88616c2d411486ed, entries=7, sequenceid=26, filesize=5.4 K 2024-11-16T02:34:51,356 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/.tmp/c/e1a56f58ba1f4119b9124158fbb52017 as hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/c/e1a56f58ba1f4119b9124158fbb52017 2024-11-16T02:34:51,362 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/c/e1a56f58ba1f4119b9124158fbb52017, entries=6, sequenceid=26, filesize=5.3 K 2024-11-16T02:34:51,362 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 28a84f7f3906829018638c93ecbbe757 in 104ms, sequenceid=26, compaction requested=false; wal=null 2024-11-16T02:34:51,363 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/0000000000000000026 2024-11-16T02:34:51,364 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,364 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,364 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:51,366 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,368 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsAfterAbortingFlush/28a84f7f3906829018638c93ecbbe757/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-16T02:34:51,368 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28a84f7f3906829018638c93ecbbe757; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74317282, jitterRate=0.10741379857063293}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:51,369 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28a84f7f3906829018638c93ecbbe757 2024-11-16T02:34:51,369 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28a84f7f3906829018638c93ecbbe757: Running coprocessor pre-open hook at 1731724491222Writing region info on filesystem at 1731724491222Initializing all the Stores at 1731724491223 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724491223Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724491223Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724491223Obtaining lock to block concurrent updates at 1731724491258 (+35 ms)Preparing flush snapshotting stores in 28a84f7f3906829018638c93ecbbe757 at 1731724491258Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731724491258Flushing stores of testReplayEditsAfterAbortingFlush,,1731724481832.28a84f7f3906829018638c93ecbbe757. at 1731724491258Flushing 28a84f7f3906829018638c93ecbbe757/a: creating writer at 1731724491258Flushing 28a84f7f3906829018638c93ecbbe757/a: appending metadata at 1731724491277 (+19 ms)Flushing 28a84f7f3906829018638c93ecbbe757/a: closing flushed file at 1731724491277Flushing 28a84f7f3906829018638c93ecbbe757/b: creating writer at 1731724491291 (+14 ms)Flushing 28a84f7f3906829018638c93ecbbe757/b: appending metadata at 1731724491304 (+13 ms)Flushing 28a84f7f3906829018638c93ecbbe757/b: closing flushed file at 1731724491304Flushing 28a84f7f3906829018638c93ecbbe757/c: creating writer at 1731724491316 (+12 ms)Flushing 28a84f7f3906829018638c93ecbbe757/c: appending metadata at 1731724491330 (+14 ms)Flushing 28a84f7f3906829018638c93ecbbe757/c: closing flushed file at 1731724491330Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b742b44: reopening flushed file at 1731724491343 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a9e5275: reopening flushed file at 1731724491349 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ebeb5fc: reopening flushed file at 1731724491355 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 28a84f7f3906829018638c93ecbbe757 in 104ms, sequenceid=26, compaction requested=false; wal=null at 1731724491362 (+7 ms)Cleaning up temporary data from old regions at 1731724491364 (+2 ms)Running coprocessor post-open hooks at 1731724491369 (+5 ms)Region opened successfully at 1731724491369 2024-11-16T02:34:51,387 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 414) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:58948 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:59672 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:52194 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:58924 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:59708 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:52182 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1182 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=325 (was 384), ProcessCount=11 (was 11), AvailableMemoryMB=4119 (was 4132) 2024-11-16T02:34:51,387 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-16T02:34:51,399 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=325, ProcessCount=11, AvailableMemoryMB=4119 2024-11-16T02:34:51,400 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-16T02:34:51,417 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:51,418 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:51,419 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:51,421 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-30807148, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/hregion-30807148, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:51,432 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-30807148/hregion-30807148.1731724491422, exclude list is [], retry=0 2024-11-16T02:34:51,435 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:51,435 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:51,435 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:51,459 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-30807148/hregion-30807148.1731724491422 2024-11-16T02:34:51,459 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131)] 2024-11-16T02:34:51,460 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 66652dc13765a383d88d904ebabed44e, NAME => 'testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43879/hbase 2024-11-16T02:34:51,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741910_1088 (size=61) 2024-11-16T02:34:51,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741910_1088 (size=61) 2024-11-16T02:34:51,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741910_1088 (size=61) 2024-11-16T02:34:51,468 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:51,469 INFO [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,471 INFO [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66652dc13765a383d88d904ebabed44e columnFamilyName a 2024-11-16T02:34:51,471 DEBUG [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:51,471 INFO [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] regionserver.HStore(327): Store=66652dc13765a383d88d904ebabed44e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:51,471 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,472 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,472 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,472 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,472 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,474 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,475 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:51,476 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 66652dc13765a383d88d904ebabed44e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71941275, jitterRate=0.07200853526592255}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T02:34:51,476 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 66652dc13765a383d88d904ebabed44e: Writing region info on filesystem at 1731724491468Initializing all the Stores at 1731724491469 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724491469Cleaning up temporary data from old regions at 1731724491472 (+3 ms)Region opened successfully at 1731724491476 (+4 ms) 2024-11-16T02:34:51,476 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 66652dc13765a383d88d904ebabed44e, disabling compactions & flushes 2024-11-16T02:34:51,476 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e. 2024-11-16T02:34:51,476 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e. 2024-11-16T02:34:51,476 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e. after waiting 0 ms 2024-11-16T02:34:51,476 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e. 2024-11-16T02:34:51,477 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e. 2024-11-16T02:34:51,477 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 66652dc13765a383d88d904ebabed44e: Waiting for close lock at 1731724491476Disabling compacts and flushes for region at 1731724491476Disabling writes for close at 1731724491476Writing region close event to WAL at 1731724491477 (+1 ms)Closed at 1731724491477 2024-11-16T02:34:51,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741909_1087 (size=175) 2024-11-16T02:34:51,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741909_1087 (size=175) 2024-11-16T02:34:51,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741909_1087 (size=175) 2024-11-16T02:34:51,482 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:51,482 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-30807148:(num 1731724491422) 2024-11-16T02:34:51,482 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:51,484 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:51,498 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484, exclude list is [], retry=0 2024-11-16T02:34:51,501 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:51,501 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:51,502 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:51,526 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484 2024-11-16T02:34:51,526 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:51,526 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 66652dc13765a383d88d904ebabed44e, NAME => 'testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:51,526 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:51,527 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,527 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,528 INFO [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,529 INFO [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66652dc13765a383d88d904ebabed44e columnFamilyName a 2024-11-16T02:34:51,529 DEBUG [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:51,529 INFO [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] regionserver.HStore(327): Store=66652dc13765a383d88d904ebabed44e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:51,530 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,530 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,531 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,532 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,532 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,534 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:51,534 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 66652dc13765a383d88d904ebabed44e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64871472, jitterRate=-0.033339738845825195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T02:34:51,535 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 66652dc13765a383d88d904ebabed44e: Writing region info on filesystem at 1731724491527Initializing all the Stores at 1731724491528 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724491528Cleaning up temporary data from old regions at 1731724491532 (+4 ms)Region opened successfully at 1731724491535 (+3 ms) 2024-11-16T02:34:51,543 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 66652dc13765a383d88d904ebabed44e, disabling compactions & flushes 2024-11-16T02:34:51,543 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e. 2024-11-16T02:34:51,543 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e. 2024-11-16T02:34:51,543 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e. after waiting 0 ms 2024-11-16T02:34:51,543 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e. 2024-11-16T02:34:51,545 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e. 2024-11-16T02:34:51,545 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e. 2024-11-16T02:34:51,545 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 66652dc13765a383d88d904ebabed44e: Waiting for close lock at 1731724491543Disabling compacts and flushes for region at 1731724491543Disabling writes for close at 1731724491543Writing region close event to WAL at 1731724491545 (+2 ms)Closed at 1731724491545 2024-11-16T02:34:51,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741911_1089 (size=1795) 2024-11-16T02:34:51,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741911_1089 (size=1795) 2024-11-16T02:34:51,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741911_1089 (size=1795) 2024-11-16T02:34:51,563 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484, size=1.8 K (1795bytes) 2024-11-16T02:34:51,563 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484 2024-11-16T02:34:51,564 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484 after 1ms 2024-11-16T02:34:51,588 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:51,588 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484 took 25ms 2024-11-16T02:34:51,590 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484 so closing down 2024-11-16T02:34:51,590 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:51,591 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731724491484.temp 2024-11-16T02:34:51,616 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/0000000000000000003-wal.1731724491484.temp 2024-11-16T02:34:51,616 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:51,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741912_1090 (size=1795) 2024-11-16T02:34:51,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741912_1090 (size=1795) 2024-11-16T02:34:51,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741912_1090 (size=1795) 2024-11-16T02:34:51,623 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/0000000000000000003-wal.1731724491484.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-16T02:34:51,624 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/0000000000000000003-wal.1731724491484.temp to hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/0000000000000000012 2024-11-16T02:34:51,624 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-16T02:34:51,624 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484, journal: Splitting hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484, size=1.8 K (1795bytes) at 1731724491563Finishing writing output for hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484 so closing down at 1731724491590 (+27 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/0000000000000000003-wal.1731724491484.temp at 1731724491616 (+26 ms)3 split writer threads finished at 1731724491616Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/0000000000000000003-wal.1731724491484.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731724491623 (+7 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/0000000000000000003-wal.1731724491484.temp to hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/0000000000000000012 at 1731724491624 (+1 ms)Processed 10 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1731724491624 2024-11-16T02:34:51,626 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724491484 to hdfs://localhost:43879/hbase/oldWALs/wal.1731724491484 2024-11-16T02:34:51,627 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/0000000000000000012 2024-11-16T02:34:51,653 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:51,929 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-16T02:34:51,929 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-16T02:34:52,025 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:52,027 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:52,039 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724492027, exclude list is [], retry=0 2024-11-16T02:34:52,041 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:52,042 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:52,042 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:52,065 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724492027 2024-11-16T02:34:52,066 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:52,066 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 66652dc13765a383d88d904ebabed44e, NAME => 'testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:52,066 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:52,066 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,066 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,069 INFO [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,071 INFO [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66652dc13765a383d88d904ebabed44e columnFamilyName a 2024-11-16T02:34:52,071 DEBUG [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:52,072 INFO [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] regionserver.HStore(327): Store=66652dc13765a383d88d904ebabed44e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:52,072 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,072 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,074 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,075 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/0000000000000000012 2024-11-16T02:34:52,102 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:52,103 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/0000000000000000012 2024-11-16T02:34:52,103 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 66652dc13765a383d88d904ebabed44e 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-16T02:34:52,121 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/.tmp/a/b7de4ff4e51343d59dc3f9d5d53f6b42 is 79, key is testDatalossWhenInputError/a:x0/1731724491535/Put/seqid=0 2024-11-16T02:34:52,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741914_1092 (size=5808) 2024-11-16T02:34:52,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741914_1092 (size=5808) 2024-11-16T02:34:52,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741914_1092 (size=5808) 2024-11-16T02:34:52,131 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/.tmp/a/b7de4ff4e51343d59dc3f9d5d53f6b42 2024-11-16T02:34:52,141 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/.tmp/a/b7de4ff4e51343d59dc3f9d5d53f6b42 as hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/a/b7de4ff4e51343d59dc3f9d5d53f6b42 2024-11-16T02:34:52,150 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/a/b7de4ff4e51343d59dc3f9d5d53f6b42, entries=10, sequenceid=12, filesize=5.7 K 2024-11-16T02:34:52,150 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 66652dc13765a383d88d904ebabed44e in 47ms, sequenceid=12, compaction requested=false; wal=null 2024-11-16T02:34:52,151 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/0000000000000000012 2024-11-16T02:34:52,152 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,152 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,155 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,158 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-16T02:34:52,159 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 66652dc13765a383d88d904ebabed44e; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69243295, jitterRate=0.03180550038814545}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T02:34:52,159 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 66652dc13765a383d88d904ebabed44e: Writing region info on filesystem at 1731724492066Initializing all the Stores at 1731724492069 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724492069Obtaining lock to block concurrent updates at 1731724492103 (+34 ms)Preparing flush snapshotting stores in 66652dc13765a383d88d904ebabed44e at 1731724492103Finished memstore snapshotting testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731724492103Flushing stores of testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e. at 1731724492103Flushing 66652dc13765a383d88d904ebabed44e/a: creating writer at 1731724492103Flushing 66652dc13765a383d88d904ebabed44e/a: appending metadata at 1731724492120 (+17 ms)Flushing 66652dc13765a383d88d904ebabed44e/a: closing flushed file at 1731724492120Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@538c672: reopening flushed file at 1731724492139 (+19 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 66652dc13765a383d88d904ebabed44e in 47ms, sequenceid=12, compaction requested=false; wal=null at 1731724492150 (+11 ms)Cleaning up temporary data from old regions at 1731724492152 (+2 ms)Region opened successfully at 1731724492159 (+7 ms) 2024-11-16T02:34:52,163 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 66652dc13765a383d88d904ebabed44e, NAME => 'testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:52,163 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731724491417.66652dc13765a383d88d904ebabed44e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:52,163 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,163 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,164 INFO [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,165 INFO [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66652dc13765a383d88d904ebabed44e columnFamilyName a 2024-11-16T02:34:52,165 DEBUG [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:52,171 DEBUG [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/a/b7de4ff4e51343d59dc3f9d5d53f6b42 2024-11-16T02:34:52,171 INFO [StoreOpener-66652dc13765a383d88d904ebabed44e-1 {}] regionserver.HStore(327): Store=66652dc13765a383d88d904ebabed44e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:52,171 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,172 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,173 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,174 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,174 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,176 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 66652dc13765a383d88d904ebabed44e 2024-11-16T02:34:52,178 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testDatalossWhenInputError/66652dc13765a383d88d904ebabed44e/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-16T02:34:52,179 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 66652dc13765a383d88d904ebabed44e; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67145326, jitterRate=5.433261394500732E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T02:34:52,179 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 66652dc13765a383d88d904ebabed44e: Writing region info on filesystem at 1731724492163Initializing all the Stores at 1731724492164 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724492164Cleaning up temporary data from old regions at 1731724492174 (+10 ms)Region opened successfully at 1731724492179 (+5 ms) 2024-11-16T02:34:52,195 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:48334 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:35606 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:49136 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:58924 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:59708 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1221085024-172.17.0.3-1731724457043:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=325 (was 325), ProcessCount=11 (was 11), AvailableMemoryMB=4094 (was 4119) 2024-11-16T02:34:52,195 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-16T02:34:52,207 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=325, ProcessCount=11, AvailableMemoryMB=4093 2024-11-16T02:34:52,207 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-16T02:34:52,220 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:52,222 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:52,222 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:52,224 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-38084639, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/hregion-38084639, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:52,237 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-38084639/hregion-38084639.1731724492225, exclude list is [], retry=0 2024-11-16T02:34:52,240 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:52,240 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:52,240 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:52,265 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-38084639/hregion-38084639.1731724492225 2024-11-16T02:34:52,265 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131)] 2024-11-16T02:34:52,265 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8a19aa740b83f6ce921329972a438574, NAME => 'testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43879/hbase 2024-11-16T02:34:52,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741916_1094 (size=63) 2024-11-16T02:34:52,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741916_1094 (size=63) 2024-11-16T02:34:52,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741916_1094 (size=63) 2024-11-16T02:34:52,277 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:52,278 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,280 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a19aa740b83f6ce921329972a438574 columnFamilyName a 2024-11-16T02:34:52,280 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:52,280 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(327): Store=8a19aa740b83f6ce921329972a438574/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:52,280 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,282 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a19aa740b83f6ce921329972a438574 columnFamilyName b 2024-11-16T02:34:52,282 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:52,282 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(327): Store=8a19aa740b83f6ce921329972a438574/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:52,282 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,283 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a19aa740b83f6ce921329972a438574 columnFamilyName c 2024-11-16T02:34:52,284 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:52,284 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(327): Store=8a19aa740b83f6ce921329972a438574/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:52,284 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,285 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,285 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,286 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,286 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,287 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:52,288 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,290 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:52,290 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8a19aa740b83f6ce921329972a438574; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58895674, jitterRate=-0.12238606810569763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:52,291 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8a19aa740b83f6ce921329972a438574: Writing region info on filesystem at 1731724492277Initializing all the Stores at 1731724492278 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724492278Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724492278Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724492278Cleaning up temporary data from old regions at 1731724492286 (+8 ms)Region opened successfully at 1731724492291 (+5 ms) 2024-11-16T02:34:52,291 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8a19aa740b83f6ce921329972a438574, disabling compactions & flushes 2024-11-16T02:34:52,291 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574. 2024-11-16T02:34:52,291 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574. 2024-11-16T02:34:52,291 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574. after waiting 0 ms 2024-11-16T02:34:52,291 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574. 2024-11-16T02:34:52,292 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574. 2024-11-16T02:34:52,292 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8a19aa740b83f6ce921329972a438574: Waiting for close lock at 1731724492291Disabling compacts and flushes for region at 1731724492291Disabling writes for close at 1731724492291Writing region close event to WAL at 1731724492292 (+1 ms)Closed at 1731724492292 2024-11-16T02:34:52,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741915_1093 (size=175) 2024-11-16T02:34:52,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741915_1093 (size=175) 2024-11-16T02:34:52,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741915_1093 (size=175) 2024-11-16T02:34:52,297 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:52,298 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-38084639:(num 1731724492225) 2024-11-16T02:34:52,298 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:52,300 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:52,312 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300, exclude list is [], retry=0 2024-11-16T02:34:52,315 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:52,315 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:52,315 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:52,343 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300 2024-11-16T02:34:52,343 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:38477:38477)] 2024-11-16T02:34:52,343 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a19aa740b83f6ce921329972a438574, NAME => 'testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:52,343 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:52,343 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,343 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,345 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,346 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a19aa740b83f6ce921329972a438574 columnFamilyName a 2024-11-16T02:34:52,346 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:52,347 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(327): Store=8a19aa740b83f6ce921329972a438574/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:52,347 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,348 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a19aa740b83f6ce921329972a438574 columnFamilyName b 2024-11-16T02:34:52,348 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:52,348 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(327): Store=8a19aa740b83f6ce921329972a438574/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:52,348 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,349 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a19aa740b83f6ce921329972a438574 columnFamilyName c 2024-11-16T02:34:52,349 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:52,350 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(327): Store=8a19aa740b83f6ce921329972a438574/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:52,350 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,351 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,352 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,353 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,353 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,353 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:52,355 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:52,356 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8a19aa740b83f6ce921329972a438574; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59563056, jitterRate=-0.1124413013458252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:52,357 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8a19aa740b83f6ce921329972a438574: Writing region info on filesystem at 1731724492344Initializing all the Stores at 1731724492345 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724492345Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724492345Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724492345Cleaning up temporary data from old regions at 1731724492353 (+8 ms)Region opened successfully at 1731724492356 (+3 ms) 2024-11-16T02:34:52,361 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731724492361/Put/seqid=0 2024-11-16T02:34:52,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741918_1096 (size=4875) 2024-11-16T02:34:52,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741918_1096 (size=4875) 2024-11-16T02:34:52,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741918_1096 (size=4875) 2024-11-16T02:34:52,771 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731724492771/Put/seqid=0 2024-11-16T02:34:52,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741919_1097 (size=4875) 2024-11-16T02:34:52,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741919_1097 (size=4875) 2024-11-16T02:34:52,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741919_1097 (size=4875) 2024-11-16T02:34:52,785 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731724492784/Put/seqid=0 2024-11-16T02:34:52,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741920_1098 (size=4875) 2024-11-16T02:34:52,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741920_1098 (size=4875) 2024-11-16T02:34:52,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741920_1098 (size=4875) 2024-11-16T02:34:52,791 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 8a19aa740b83f6ce921329972a438574/a 2024-11-16T02:34:52,794 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-16T02:34:52,794 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-16T02:34:52,794 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 8a19aa740b83f6ce921329972a438574/a 2024-11-16T02:34:52,797 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-16T02:34:52,798 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-16T02:34:52,798 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 8a19aa740b83f6ce921329972a438574/a 2024-11-16T02:34:52,801 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-16T02:34:52,801 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-16T02:34:52,801 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8a19aa740b83f6ce921329972a438574 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-16T02:34:52,817 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/.tmp/a/0a9ce6ef6a61487c927710755dd2a913 is 55, key is testCompactedBulkLoadedFiles/a:a/1731724492357/Put/seqid=0 2024-11-16T02:34:52,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741921_1099 (size=5107) 2024-11-16T02:34:52,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741921_1099 (size=5107) 2024-11-16T02:34:52,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741921_1099 (size=5107) 2024-11-16T02:34:52,825 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/.tmp/a/0a9ce6ef6a61487c927710755dd2a913 2024-11-16T02:34:52,831 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/.tmp/a/0a9ce6ef6a61487c927710755dd2a913 as hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/0a9ce6ef6a61487c927710755dd2a913 2024-11-16T02:34:52,836 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/0a9ce6ef6a61487c927710755dd2a913, entries=1, sequenceid=4, filesize=5.0 K 2024-11-16T02:34:52,837 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 8a19aa740b83f6ce921329972a438574 in 36ms, sequenceid=4, compaction requested=false 2024-11-16T02:34:52,837 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8a19aa740b83f6ce921329972a438574: 2024-11-16T02:34:52,839 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_ 2024-11-16T02:34:52,840 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/6094377b6ae84cbb9b208b27a58ef250_SeqId_4_ 2024-11-16T02:34:52,841 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/757956c8f53e40e49309ee13f48848f6_SeqId_4_ 2024-11-16T02:34:52,841 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile0 into 8a19aa740b83f6ce921329972a438574/a as hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_ - updating store file list. 2024-11-16T02:34:52,845 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-16T02:34:52,845 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_ into 8a19aa740b83f6ce921329972a438574/a 2024-11-16T02:34:52,845 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile0 into 8a19aa740b83f6ce921329972a438574/a (new location: hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_) 2024-11-16T02:34:52,846 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile1 into 8a19aa740b83f6ce921329972a438574/a as hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/6094377b6ae84cbb9b208b27a58ef250_SeqId_4_ - updating store file list. 2024-11-16T02:34:52,850 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6094377b6ae84cbb9b208b27a58ef250_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-16T02:34:52,850 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/6094377b6ae84cbb9b208b27a58ef250_SeqId_4_ into 8a19aa740b83f6ce921329972a438574/a 2024-11-16T02:34:52,850 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile1 into 8a19aa740b83f6ce921329972a438574/a (new location: hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/6094377b6ae84cbb9b208b27a58ef250_SeqId_4_) 2024-11-16T02:34:52,851 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile2 into 8a19aa740b83f6ce921329972a438574/a as hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/757956c8f53e40e49309ee13f48848f6_SeqId_4_ - updating store file list. 2024-11-16T02:34:52,854 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 757956c8f53e40e49309ee13f48848f6_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-16T02:34:52,855 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/757956c8f53e40e49309ee13f48848f6_SeqId_4_ into 8a19aa740b83f6ce921329972a438574/a 2024-11-16T02:34:52,855 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43879/hbase/testCompactedBulkLoadedFiles/hfile2 into 8a19aa740b83f6ce921329972a438574/a (new location: hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/757956c8f53e40e49309ee13f48848f6_SeqId_4_) 2024-11-16T02:34:52,862 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T02:34:52,862 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8a19aa740b83f6ce921329972a438574/a is initiating major compaction (all files) 2024-11-16T02:34:52,862 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8a19aa740b83f6ce921329972a438574/a in testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574. 2024-11-16T02:34:52,862 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/0a9ce6ef6a61487c927710755dd2a913, hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_, hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/6094377b6ae84cbb9b208b27a58ef250_SeqId_4_, hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/757956c8f53e40e49309ee13f48848f6_SeqId_4_] into tmpdir=hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/.tmp, totalSize=19.3 K 2024-11-16T02:34:52,863 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0a9ce6ef6a61487c927710755dd2a913, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731724492357 2024-11-16T02:34:52,863 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-16T02:34:52,863 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6094377b6ae84cbb9b208b27a58ef250_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-16T02:34:52,864 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 757956c8f53e40e49309ee13f48848f6_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-16T02:34:52,875 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/.tmp/a/75b722e9fabd4c5ca0d684b717787854 is 55, key is testCompactedBulkLoadedFiles/a:a/1731724492357/Put/seqid=0 2024-11-16T02:34:52,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741922_1100 (size=6154) 2024-11-16T02:34:52,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741922_1100 (size=6154) 2024-11-16T02:34:52,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741922_1100 (size=6154) 2024-11-16T02:34:52,888 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/.tmp/a/75b722e9fabd4c5ca0d684b717787854 as hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/75b722e9fabd4c5ca0d684b717787854 2024-11-16T02:34:52,895 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 8a19aa740b83f6ce921329972a438574/a of 8a19aa740b83f6ce921329972a438574 into 75b722e9fabd4c5ca0d684b717787854(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T02:34:52,895 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8a19aa740b83f6ce921329972a438574: 2024-11-16T02:34:52,895 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-16T02:34:52,895 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-16T02:34:52,932 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300, size=0 (0bytes) 2024-11-16T02:34:52,932 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300 might be still open, length is 0 2024-11-16T02:34:52,932 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300 2024-11-16T02:34:52,933 WARN [IPC Server handler 1 on default port 43879 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-16T02:34:52,933 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300 after 1ms 2024-11-16T02:34:53,641 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:49190 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33993:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49190 dst: /127.0.0.1:33993 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33993 remote=/127.0.0.1:49190]. Total timeout mills is 60000, 59254 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:53,641 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:48364 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:39531:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48364 dst: /127.0.0.1:39531 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:53,641 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:35644 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35644 dst: /127.0.0.1:40973 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:53,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741917_1101 (size=1794) 2024-11-16T02:34:53,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741917_1101 (size=1794) 2024-11-16T02:34:53,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741917_1101 (size=1794) 2024-11-16T02:34:56,630 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T02:34:56,935 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300 after 4003ms 2024-11-16T02:34:56,968 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:56,968 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300 took 4036ms 2024-11-16T02:34:56,970 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300; continuing. 2024-11-16T02:34:56,970 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300 so closing down 2024-11-16T02:34:56,970 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:56,971 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731724492300.temp 2024-11-16T02:34:56,995 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/0000000000000000003-wal.1731724492300.temp 2024-11-16T02:34:56,996 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:57,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741923_1102 (size=800) 2024-11-16T02:34:57,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741923_1102 (size=800) 2024-11-16T02:34:57,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741923_1102 (size=800) 2024-11-16T02:34:57,003 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/0000000000000000003-wal.1731724492300.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-16T02:34:57,004 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/0000000000000000003-wal.1731724492300.temp to hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/0000000000000000008 2024-11-16T02:34:57,004 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300, size=0, length=0, corrupted=false, cancelled=false 2024-11-16T02:34:57,004 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300, journal: Splitting hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300, size=0 (0bytes) at 1731724492932Finishing writing output for hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300 so closing down at 1731724496970 (+4038 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/0000000000000000003-wal.1731724492300.temp at 1731724496995 (+25 ms)3 split writer threads finished at 1731724496996 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/0000000000000000003-wal.1731724492300.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731724497003 (+7 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/0000000000000000003-wal.1731724492300.temp to hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/0000000000000000008 at 1731724497004 (+1 ms)Processed 5 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300, size=0, length=0, corrupted=false, cancelled=false at 1731724497004 2024-11-16T02:34:57,005 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300 to hdfs://localhost:43879/hbase/oldWALs/wal.1731724492300 2024-11-16T02:34:57,006 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/0000000000000000008 2024-11-16T02:34:57,006 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:57,008 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:57,022 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724497008, exclude list is [], retry=0 2024-11-16T02:34:57,024 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:57,024 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:57,025 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:57,050 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724497008 2024-11-16T02:34:57,051 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:57,051 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a19aa740b83f6ce921329972a438574, NAME => 'testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:57,051 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:57,051 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:57,051 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:57,052 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:57,053 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a19aa740b83f6ce921329972a438574 columnFamilyName a 2024-11-16T02:34:57,053 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:57,059 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/0a9ce6ef6a61487c927710755dd2a913 2024-11-16T02:34:57,063 DEBUG [StoreFileOpener-8a19aa740b83f6ce921329972a438574-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6094377b6ae84cbb9b208b27a58ef250_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-16T02:34:57,063 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/6094377b6ae84cbb9b208b27a58ef250_SeqId_4_ 2024-11-16T02:34:57,067 DEBUG [StoreFileOpener-8a19aa740b83f6ce921329972a438574-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 757956c8f53e40e49309ee13f48848f6_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-16T02:34:57,067 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/757956c8f53e40e49309ee13f48848f6_SeqId_4_ 2024-11-16T02:34:57,070 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/75b722e9fabd4c5ca0d684b717787854 2024-11-16T02:34:57,073 DEBUG [StoreFileOpener-8a19aa740b83f6ce921329972a438574-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-16T02:34:57,073 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_ 2024-11-16T02:34:57,074 WARN [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/0a9ce6ef6a61487c927710755dd2a913 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7a4d7efe 2024-11-16T02:34:57,074 WARN [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/6094377b6ae84cbb9b208b27a58ef250_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7a4d7efe 2024-11-16T02:34:57,074 WARN [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/757956c8f53e40e49309ee13f48848f6_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7a4d7efe 2024-11-16T02:34:57,074 WARN [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7a4d7efe 2024-11-16T02:34:57,074 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/0a9ce6ef6a61487c927710755dd2a913, hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/6094377b6ae84cbb9b208b27a58ef250_SeqId_4_, hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/757956c8f53e40e49309ee13f48848f6_SeqId_4_, hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_] to archive 2024-11-16T02:34:57,075 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T02:34:57,076 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/0a9ce6ef6a61487c927710755dd2a913 to hdfs://localhost:43879/hbase/archive/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/0a9ce6ef6a61487c927710755dd2a913 2024-11-16T02:34:57,077 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/6094377b6ae84cbb9b208b27a58ef250_SeqId_4_ to hdfs://localhost:43879/hbase/archive/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/6094377b6ae84cbb9b208b27a58ef250_SeqId_4_ 2024-11-16T02:34:57,078 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/757956c8f53e40e49309ee13f48848f6_SeqId_4_ to hdfs://localhost:43879/hbase/archive/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/757956c8f53e40e49309ee13f48848f6_SeqId_4_ 2024-11-16T02:34:57,079 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_ to hdfs://localhost:43879/hbase/archive/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/a/d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_ 2024-11-16T02:34:57,079 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(327): Store=8a19aa740b83f6ce921329972a438574/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:57,079 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:57,080 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a19aa740b83f6ce921329972a438574 columnFamilyName b 2024-11-16T02:34:57,080 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:57,081 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(327): Store=8a19aa740b83f6ce921329972a438574/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:57,081 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:57,081 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a19aa740b83f6ce921329972a438574 columnFamilyName c 2024-11-16T02:34:57,081 DEBUG [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:57,082 INFO [StoreOpener-8a19aa740b83f6ce921329972a438574-1 {}] regionserver.HStore(327): Store=8a19aa740b83f6ce921329972a438574/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:57,082 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:57,082 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:57,083 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:57,084 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/0000000000000000008 2024-11-16T02:34:57,109 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:57,111 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 8a19aa740b83f6ce921329972a438574 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "8a19aa740b83f6ce921329972a438574" family_name: "a" compaction_input: "0a9ce6ef6a61487c927710755dd2a913" compaction_input: "d1bedeac49f94f9fbf9eb90e7376ce23_SeqId_4_" compaction_input: "6094377b6ae84cbb9b208b27a58ef250_SeqId_4_" compaction_input: "757956c8f53e40e49309ee13f48848f6_SeqId_4_" compaction_output: "75b722e9fabd4c5ca0d684b717787854" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-16T02:34:57,111 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-16T02:34:57,111 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/0000000000000000008 2024-11-16T02:34:57,112 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/0000000000000000008 2024-11-16T02:34:57,113 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:57,113 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:57,114 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:57,115 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8a19aa740b83f6ce921329972a438574 2024-11-16T02:34:57,117 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testCompactedBulkLoadedFiles/8a19aa740b83f6ce921329972a438574/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T02:34:57,118 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8a19aa740b83f6ce921329972a438574; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71063194, jitterRate=0.0589241087436676}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:57,118 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8a19aa740b83f6ce921329972a438574: Writing region info on filesystem at 1731724497051Initializing all the Stores at 1731724497052 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724497052Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724497052Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724497052Cleaning up temporary data from old regions at 1731724497113 (+61 ms)Region opened successfully at 1731724497118 (+5 ms) 2024-11-16T02:34:57,120 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8a19aa740b83f6ce921329972a438574, disabling compactions & flushes 2024-11-16T02:34:57,120 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574. 2024-11-16T02:34:57,120 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574. 2024-11-16T02:34:57,120 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574. after waiting 0 ms 2024-11-16T02:34:57,120 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574. 2024-11-16T02:34:57,121 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731724492221.8a19aa740b83f6ce921329972a438574. 2024-11-16T02:34:57,121 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8a19aa740b83f6ce921329972a438574: Waiting for close lock at 1731724497120Disabling compacts and flushes for region at 1731724497120Disabling writes for close at 1731724497120Writing region close event to WAL at 1731724497121 (+1 ms)Closed at 1731724497121 2024-11-16T02:34:57,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741924_1103 (size=175) 2024-11-16T02:34:57,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741924_1103 (size=175) 2024-11-16T02:34:57,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741924_1103 (size=175) 2024-11-16T02:34:57,125 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:57,125 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731724497008) 2024-11-16T02:34:57,137 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=440 (was 425) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:43879 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:43879 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-56928516_22 at /127.0.0.1:49226 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-56928516_22 at /127.0.0.1:48454 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-56928516_22 at /127.0.0.1:35714 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=307 (was 325), ProcessCount=11 (was 11), AvailableMemoryMB=4087 (was 4093) 2024-11-16T02:34:57,138 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-16T02:34:57,151 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=440, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=307, ProcessCount=11, AvailableMemoryMB=4086 2024-11-16T02:34:57,151 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-16T02:34:57,167 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:57,168 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T02:34:57,169 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T02:34:57,171 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-01675947, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/hregion-01675947, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:57,184 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-01675947/hregion-01675947.1731724497171, exclude list is [], retry=0 2024-11-16T02:34:57,187 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:57,187 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:57,187 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:57,211 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-01675947/hregion-01675947.1731724497171 2024-11-16T02:34:57,211 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:34:57,212 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 159bcfb9a61891ff7e6db58af5cdad8b, NAME => 'testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43879/hbase 2024-11-16T02:34:57,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741926_1105 (size=67) 2024-11-16T02:34:57,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741926_1105 (size=67) 2024-11-16T02:34:57,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741926_1105 (size=67) 2024-11-16T02:34:57,220 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:57,221 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,223 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159bcfb9a61891ff7e6db58af5cdad8b columnFamilyName a 2024-11-16T02:34:57,223 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:57,223 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(327): Store=159bcfb9a61891ff7e6db58af5cdad8b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:57,223 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,224 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159bcfb9a61891ff7e6db58af5cdad8b columnFamilyName b 2024-11-16T02:34:57,224 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:57,225 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(327): Store=159bcfb9a61891ff7e6db58af5cdad8b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:57,225 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,226 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159bcfb9a61891ff7e6db58af5cdad8b columnFamilyName c 2024-11-16T02:34:57,226 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:57,226 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(327): Store=159bcfb9a61891ff7e6db58af5cdad8b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:57,227 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,227 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,227 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,228 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,228 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,229 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:57,230 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,232 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T02:34:57,233 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 159bcfb9a61891ff7e6db58af5cdad8b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61866711, jitterRate=-0.07811416685581207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:57,233 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 159bcfb9a61891ff7e6db58af5cdad8b: Writing region info on filesystem at 1731724497220Initializing all the Stores at 1731724497221 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724497221Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724497221Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724497221Cleaning up temporary data from old regions at 1731724497228 (+7 ms)Region opened successfully at 1731724497233 (+5 ms) 2024-11-16T02:34:57,233 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 159bcfb9a61891ff7e6db58af5cdad8b, disabling compactions & flushes 2024-11-16T02:34:57,233 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:34:57,233 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:34:57,233 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. after waiting 0 ms 2024-11-16T02:34:57,233 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:34:57,234 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:34:57,234 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 159bcfb9a61891ff7e6db58af5cdad8b: Waiting for close lock at 1731724497233Disabling compacts and flushes for region at 1731724497233Disabling writes for close at 1731724497233Writing region close event to WAL at 1731724497234 (+1 ms)Closed at 1731724497234 2024-11-16T02:34:57,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741925_1104 (size=175) 2024-11-16T02:34:57,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741925_1104 (size=175) 2024-11-16T02:34:57,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741925_1104 (size=175) 2024-11-16T02:34:57,239 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:34:57,239 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-01675947:(num 1731724497171) 2024-11-16T02:34:57,239 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:57,241 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:57,253 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241, exclude list is [], retry=0 2024-11-16T02:34:57,255 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:57,256 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:57,256 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:57,280 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241 2024-11-16T02:34:57,280 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:41131:41131)] 2024-11-16T02:34:57,280 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 159bcfb9a61891ff7e6db58af5cdad8b, NAME => 'testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:57,280 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:57,281 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,281 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,282 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,283 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159bcfb9a61891ff7e6db58af5cdad8b columnFamilyName a 2024-11-16T02:34:57,283 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:57,283 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(327): Store=159bcfb9a61891ff7e6db58af5cdad8b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:57,283 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,284 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159bcfb9a61891ff7e6db58af5cdad8b columnFamilyName b 2024-11-16T02:34:57,284 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:57,284 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(327): Store=159bcfb9a61891ff7e6db58af5cdad8b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:57,285 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,285 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159bcfb9a61891ff7e6db58af5cdad8b columnFamilyName c 2024-11-16T02:34:57,285 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:57,286 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(327): Store=159bcfb9a61891ff7e6db58af5cdad8b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:57,286 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,286 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,287 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,288 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,288 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,288 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:57,289 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,290 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 159bcfb9a61891ff7e6db58af5cdad8b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75431990, jitterRate=0.12402424216270447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:57,290 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 159bcfb9a61891ff7e6db58af5cdad8b: Writing region info on filesystem at 1731724497281Initializing all the Stores at 1731724497282 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724497282Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724497282Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724497282Cleaning up temporary data from old regions at 1731724497288 (+6 ms)Region opened successfully at 1731724497290 (+2 ms) 2024-11-16T02:34:57,297 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 159bcfb9a61891ff7e6db58af5cdad8b 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-16T02:34:57,311 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/a/e54b7f26e52b479c8153f622b6f0f0ce is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731724497290/Put/seqid=0 2024-11-16T02:34:57,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741928_1107 (size=5958) 2024-11-16T02:34:57,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741928_1107 (size=5958) 2024-11-16T02:34:57,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741928_1107 (size=5958) 2024-11-16T02:34:57,318 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/a/e54b7f26e52b479c8153f622b6f0f0ce 2024-11-16T02:34:57,322 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/a/e54b7f26e52b479c8153f622b6f0f0ce as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/a/e54b7f26e52b479c8153f622b6f0f0ce 2024-11-16T02:34:57,327 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/a/e54b7f26e52b479c8153f622b6f0f0ce, entries=10, sequenceid=13, filesize=5.8 K 2024-11-16T02:34:57,328 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 159bcfb9a61891ff7e6db58af5cdad8b in 31ms, sequenceid=13, compaction requested=false 2024-11-16T02:34:57,328 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 159bcfb9a61891ff7e6db58af5cdad8b: 2024-11-16T02:34:57,343 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 159bcfb9a61891ff7e6db58af5cdad8b, disabling compactions & flushes 2024-11-16T02:34:57,343 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:34:57,343 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:34:57,343 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. after waiting 0 ms 2024-11-16T02:34:57,343 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:34:57,344 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:34:57,344 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:34:57,344 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 159bcfb9a61891ff7e6db58af5cdad8b: Waiting for close lock at 1731724497343Disabling compacts and flushes for region at 1731724497343Disabling writes for close at 1731724497343Writing region close event to WAL at 1731724497344 (+1 ms)Closed at 1731724497344 2024-11-16T02:34:57,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741927_1106 (size=6243) 2024-11-16T02:34:57,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741927_1106 (size=6243) 2024-11-16T02:34:57,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741927_1106 (size=6243) 2024-11-16T02:34:57,366 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241, size=6.1 K (6243bytes) 2024-11-16T02:34:57,367 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241 2024-11-16T02:34:57,367 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241 after 0ms 2024-11-16T02:34:57,392 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:57,392 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241 took 26ms 2024-11-16T02:34:57,394 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241 so closing down 2024-11-16T02:34:57,394 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:34:57,395 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731724497241.temp 2024-11-16T02:34:57,419 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000003-wal.1731724497241.temp 2024-11-16T02:34:57,420 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:34:57,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741929_1108 (size=5575) 2024-11-16T02:34:57,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741929_1108 (size=5575) 2024-11-16T02:34:57,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741929_1108 (size=5575) 2024-11-16T02:34:57,426 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000003-wal.1731724497241.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-16T02:34:57,427 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000003-wal.1731724497241.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000035 2024-11-16T02:34:57,427 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-16T02:34:57,427 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241, journal: Splitting hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241, size=6.1 K (6243bytes) at 1731724497366Finishing writing output for hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241 so closing down at 1731724497394 (+28 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000003-wal.1731724497241.temp at 1731724497419 (+25 ms)3 split writer threads finished at 1731724497420 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000003-wal.1731724497241.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731724497426 (+6 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000003-wal.1731724497241.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000035 at 1731724497427 (+1 ms)Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1731724497427 2024-11-16T02:34:57,429 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497241 to hdfs://localhost:43879/hbase/oldWALs/wal.1731724497241 2024-11-16T02:34:57,430 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000035 2024-11-16T02:34:57,430 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:34:57,432 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:34:57,445 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432, exclude list is [], retry=0 2024-11-16T02:34:57,448 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:34:57,449 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:34:57,449 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:34:57,473 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432 2024-11-16T02:34:57,473 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:39667:39667),(127.0.0.1/127.0.0.1:38477:38477)] 2024-11-16T02:34:57,474 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 159bcfb9a61891ff7e6db58af5cdad8b, NAME => 'testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T02:34:57,474 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:34:57,474 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,474 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,475 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,476 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159bcfb9a61891ff7e6db58af5cdad8b columnFamilyName a 2024-11-16T02:34:57,476 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:57,481 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/a/e54b7f26e52b479c8153f622b6f0f0ce 2024-11-16T02:34:57,481 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(327): Store=159bcfb9a61891ff7e6db58af5cdad8b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:57,481 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,482 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159bcfb9a61891ff7e6db58af5cdad8b columnFamilyName b 2024-11-16T02:34:57,482 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:57,482 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(327): Store=159bcfb9a61891ff7e6db58af5cdad8b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:57,482 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,483 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159bcfb9a61891ff7e6db58af5cdad8b columnFamilyName c 2024-11-16T02:34:57,483 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:34:57,483 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(327): Store=159bcfb9a61891ff7e6db58af5cdad8b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:34:57,483 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,484 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,486 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,486 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000035 2024-11-16T02:34:57,512 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:34:57,513 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000035 2024-11-16T02:34:57,513 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 159bcfb9a61891ff7e6db58af5cdad8b 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-16T02:34:57,526 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/b/3869deb266f54fed86b9c9ed337f7ffc is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731724497328/Put/seqid=0 2024-11-16T02:34:57,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741931_1110 (size=5958) 2024-11-16T02:34:57,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741931_1110 (size=5958) 2024-11-16T02:34:57,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741931_1110 (size=5958) 2024-11-16T02:34:57,533 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/b/3869deb266f54fed86b9c9ed337f7ffc 2024-11-16T02:34:57,551 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/c/f23874462e6c48aa96c47645d0534301 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731724497335/Put/seqid=0 2024-11-16T02:34:57,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741932_1111 (size=5958) 2024-11-16T02:34:57,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741932_1111 (size=5958) 2024-11-16T02:34:57,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741932_1111 (size=5958) 2024-11-16T02:34:57,561 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/c/f23874462e6c48aa96c47645d0534301 2024-11-16T02:34:57,566 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/b/3869deb266f54fed86b9c9ed337f7ffc as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/b/3869deb266f54fed86b9c9ed337f7ffc 2024-11-16T02:34:57,571 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/b/3869deb266f54fed86b9c9ed337f7ffc, entries=10, sequenceid=35, filesize=5.8 K 2024-11-16T02:34:57,572 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/c/f23874462e6c48aa96c47645d0534301 as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/c/f23874462e6c48aa96c47645d0534301 2024-11-16T02:34:57,577 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/c/f23874462e6c48aa96c47645d0534301, entries=10, sequenceid=35, filesize=5.8 K 2024-11-16T02:34:57,577 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 159bcfb9a61891ff7e6db58af5cdad8b in 64ms, sequenceid=35, compaction requested=false; wal=null 2024-11-16T02:34:57,578 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000035 2024-11-16T02:34:57,579 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,579 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,579 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:34:57,580 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:34:57,582 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-16T02:34:57,583 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 159bcfb9a61891ff7e6db58af5cdad8b; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66116906, jitterRate=-0.014781326055526733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:34:57,583 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 159bcfb9a61891ff7e6db58af5cdad8b: Writing region info on filesystem at 1731724497474Initializing all the Stores at 1731724497475 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724497475Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724497475Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724497475Obtaining lock to block concurrent updates at 1731724497513 (+38 ms)Preparing flush snapshotting stores in 159bcfb9a61891ff7e6db58af5cdad8b at 1731724497513Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731724497513Flushing stores of testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. at 1731724497513Flushing 159bcfb9a61891ff7e6db58af5cdad8b/b: creating writer at 1731724497513Flushing 159bcfb9a61891ff7e6db58af5cdad8b/b: appending metadata at 1731724497525 (+12 ms)Flushing 159bcfb9a61891ff7e6db58af5cdad8b/b: closing flushed file at 1731724497525Flushing 159bcfb9a61891ff7e6db58af5cdad8b/c: creating writer at 1731724497537 (+12 ms)Flushing 159bcfb9a61891ff7e6db58af5cdad8b/c: appending metadata at 1731724497551 (+14 ms)Flushing 159bcfb9a61891ff7e6db58af5cdad8b/c: closing flushed file at 1731724497551Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30dd54cb: reopening flushed file at 1731724497565 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cf5b2f7: reopening flushed file at 1731724497571 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 159bcfb9a61891ff7e6db58af5cdad8b in 64ms, sequenceid=35, compaction requested=false; wal=null at 1731724497577 (+6 ms)Cleaning up temporary data from old regions at 1731724497579 (+2 ms)Region opened successfully at 1731724497583 (+4 ms) 2024-11-16T02:34:57,644 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432, size=0 (0bytes) 2024-11-16T02:34:57,644 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432 might be still open, length is 0 2024-11-16T02:34:57,644 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432 2024-11-16T02:34:57,644 WARN [IPC Server handler 3 on default port 43879 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-16T02:34:57,644 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432 after 0ms 2024-11-16T02:34:59,644 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:49300 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33993:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49300 dst: /127.0.0.1:33993 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33993 remote=/127.0.0.1:49300]. Total timeout mills is 60000, 57965 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:59,645 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:48504 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:39531:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48504 dst: /127.0.0.1:39531 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:59,645 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_464715242_22 at /127.0.0.1:35788 [Receiving block BP-1221085024-172.17.0.3-1731724457043:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35788 dst: /127.0.0.1:40973 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:34:59,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741930_1112 (size=5567) 2024-11-16T02:34:59,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741930_1112 (size=5567) 2024-11-16T02:34:59,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741930_1112 (size=5567) 2024-11-16T02:35:01,645 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432 after 4001ms 2024-11-16T02:35:01,712 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:35:01,712 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432 took 4069ms 2024-11-16T02:35:01,714 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432; continuing. 2024-11-16T02:35:01,714 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432 so closing down 2024-11-16T02:35:01,714 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T02:35:01,716 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731724497432.temp 2024-11-16T02:35:01,742 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000037-wal.1731724497432.temp 2024-11-16T02:35:01,742 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T02:35:01,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741933_1113 (size=5575) 2024-11-16T02:35:01,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741933_1113 (size=5575) 2024-11-16T02:35:01,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741933_1113 (size=5575) 2024-11-16T02:35:01,748 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000037-wal.1731724497432.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-16T02:35:01,749 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000037-wal.1731724497432.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000066 2024-11-16T02:35:01,749 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432, size=0, length=0, corrupted=false, cancelled=false 2024-11-16T02:35:01,749 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432, journal: Splitting hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432, size=0 (0bytes) at 1731724497644Finishing writing output for hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432 so closing down at 1731724501714 (+4070 ms)Creating recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000037-wal.1731724497432.temp at 1731724501742 (+28 ms)3 split writer threads finished at 1731724501742Closed recovered edits writer path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000037-wal.1731724497432.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731724501748 (+6 ms)Rename recovered edits hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000037-wal.1731724497432.temp to hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000066 at 1731724501749 (+1 ms)Processed 30 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432, size=0, length=0, corrupted=false, cancelled=false at 1731724501749 2024-11-16T02:35:01,751 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432 to hdfs://localhost:43879/hbase/oldWALs/wal.1731724497432 2024-11-16T02:35:01,751 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000066 2024-11-16T02:35:01,752 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T02:35:01,753 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43879/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166, archiveDir=hdfs://localhost:43879/hbase/oldWALs, maxLogs=32 2024-11-16T02:35:01,767 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724501754, exclude list is [], retry=0 2024-11-16T02:35:01,769 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33993,DS-eabca314-d433-4de7-9a38-1d7d16e3081f,DISK] 2024-11-16T02:35:01,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39531,DS-5e70e789-2331-4a70-8c9c-b6f6fedc6586,DISK] 2024-11-16T02:35:01,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40973,DS-14c4bd34-ae11-4246-a0cc-a904aa90a70b,DISK] 2024-11-16T02:35:01,794 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724501754 2024-11-16T02:35:01,795 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41131:41131),(127.0.0.1/127.0.0.1:38477:38477),(127.0.0.1/127.0.0.1:39667:39667)] 2024-11-16T02:35:01,795 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T02:35:01,796 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:35:01,797 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159bcfb9a61891ff7e6db58af5cdad8b columnFamilyName a 2024-11-16T02:35:01,797 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:35:01,801 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/a/e54b7f26e52b479c8153f622b6f0f0ce 2024-11-16T02:35:01,801 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(327): Store=159bcfb9a61891ff7e6db58af5cdad8b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:35:01,801 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:35:01,802 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159bcfb9a61891ff7e6db58af5cdad8b columnFamilyName b 2024-11-16T02:35:01,802 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:35:01,807 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/b/3869deb266f54fed86b9c9ed337f7ffc 2024-11-16T02:35:01,807 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(327): Store=159bcfb9a61891ff7e6db58af5cdad8b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:35:01,808 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:35:01,808 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159bcfb9a61891ff7e6db58af5cdad8b columnFamilyName c 2024-11-16T02:35:01,808 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T02:35:01,814 DEBUG [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/c/f23874462e6c48aa96c47645d0534301 2024-11-16T02:35:01,814 INFO [StoreOpener-159bcfb9a61891ff7e6db58af5cdad8b-1 {}] regionserver.HStore(327): Store=159bcfb9a61891ff7e6db58af5cdad8b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T02:35:01,815 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:35:01,815 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:35:01,817 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:35:01,818 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000066 2024-11-16T02:35:01,841 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-16T02:35:01,845 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000066 2024-11-16T02:35:01,845 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 159bcfb9a61891ff7e6db58af5cdad8b 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-16T02:35:01,858 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/a/65273979ae2f4bb6bbbca024fe87d3ab is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731724497590/Put/seqid=0 2024-11-16T02:35:01,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741935_1115 (size=5958) 2024-11-16T02:35:01,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741935_1115 (size=5958) 2024-11-16T02:35:01,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741935_1115 (size=5958) 2024-11-16T02:35:01,864 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/a/65273979ae2f4bb6bbbca024fe87d3ab 2024-11-16T02:35:01,882 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/b/fd07ca8212e64c0bb2653711470dd330 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731724497597/Put/seqid=0 2024-11-16T02:35:01,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741936_1116 (size=5958) 2024-11-16T02:35:01,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741936_1116 (size=5958) 2024-11-16T02:35:01,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741936_1116 (size=5958) 2024-11-16T02:35:01,888 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/b/fd07ca8212e64c0bb2653711470dd330 2024-11-16T02:35:01,909 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/c/761b165ad97248589697ea2860fcb26e is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731724497603/Put/seqid=0 2024-11-16T02:35:01,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741937_1117 (size=5958) 2024-11-16T02:35:01,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741937_1117 (size=5958) 2024-11-16T02:35:01,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741937_1117 (size=5958) 2024-11-16T02:35:01,916 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/c/761b165ad97248589697ea2860fcb26e 2024-11-16T02:35:01,921 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/a/65273979ae2f4bb6bbbca024fe87d3ab as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/a/65273979ae2f4bb6bbbca024fe87d3ab 2024-11-16T02:35:01,926 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/a/65273979ae2f4bb6bbbca024fe87d3ab, entries=10, sequenceid=66, filesize=5.8 K 2024-11-16T02:35:01,927 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/b/fd07ca8212e64c0bb2653711470dd330 as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/b/fd07ca8212e64c0bb2653711470dd330 2024-11-16T02:35:01,929 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-16T02:35:01,929 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-16T02:35:01,930 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-16T02:35:01,930 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-16T02:35:01,931 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/b/fd07ca8212e64c0bb2653711470dd330, entries=10, sequenceid=66, filesize=5.8 K 2024-11-16T02:35:01,932 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/.tmp/c/761b165ad97248589697ea2860fcb26e as hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/c/761b165ad97248589697ea2860fcb26e 2024-11-16T02:35:01,937 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/c/761b165ad97248589697ea2860fcb26e, entries=10, sequenceid=66, filesize=5.8 K 2024-11-16T02:35:01,937 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 159bcfb9a61891ff7e6db58af5cdad8b in 92ms, sequenceid=66, compaction requested=false; wal=null 2024-11-16T02:35:01,938 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/0000000000000000066 2024-11-16T02:35:01,939 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:35:01,939 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:35:01,940 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T02:35:01,941 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 159bcfb9a61891ff7e6db58af5cdad8b 2024-11-16T02:35:01,943 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/hbase/data/default/testReplayEditsWrittenViaHRegion/159bcfb9a61891ff7e6db58af5cdad8b/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-16T02:35:01,944 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 159bcfb9a61891ff7e6db58af5cdad8b; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60325294, jitterRate=-0.10108307003974915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T02:35:01,944 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 159bcfb9a61891ff7e6db58af5cdad8b: Writing region info on filesystem at 1731724501795Initializing all the Stores at 1731724501796 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724501796Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724501796Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731724501796Obtaining lock to block concurrent updates at 1731724501845 (+49 ms)Preparing flush snapshotting stores in 159bcfb9a61891ff7e6db58af5cdad8b at 1731724501845Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731724501845Flushing stores of testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. at 1731724501845Flushing 159bcfb9a61891ff7e6db58af5cdad8b/a: creating writer at 1731724501845Flushing 159bcfb9a61891ff7e6db58af5cdad8b/a: appending metadata at 1731724501857 (+12 ms)Flushing 159bcfb9a61891ff7e6db58af5cdad8b/a: closing flushed file at 1731724501857Flushing 159bcfb9a61891ff7e6db58af5cdad8b/b: creating writer at 1731724501868 (+11 ms)Flushing 159bcfb9a61891ff7e6db58af5cdad8b/b: appending metadata at 1731724501881 (+13 ms)Flushing 159bcfb9a61891ff7e6db58af5cdad8b/b: closing flushed file at 1731724501881Flushing 159bcfb9a61891ff7e6db58af5cdad8b/c: creating writer at 1731724501893 (+12 ms)Flushing 159bcfb9a61891ff7e6db58af5cdad8b/c: appending metadata at 1731724501909 (+16 ms)Flushing 159bcfb9a61891ff7e6db58af5cdad8b/c: closing flushed file at 1731724501909Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62911f50: reopening flushed file at 1731724501920 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bd6c9fd: reopening flushed file at 1731724501926 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@819f59c: reopening flushed file at 1731724501931 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 159bcfb9a61891ff7e6db58af5cdad8b in 92ms, sequenceid=66, compaction requested=false; wal=null at 1731724501937 (+6 ms)Cleaning up temporary data from old regions at 1731724501939 (+2 ms)Region opened successfully at 1731724501944 (+5 ms) 2024-11-16T02:35:01,956 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 159bcfb9a61891ff7e6db58af5cdad8b, disabling compactions & flushes 2024-11-16T02:35:01,957 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:35:01,957 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:35:01,957 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. after waiting 0 ms 2024-11-16T02:35:01,957 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:35:01,959 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731724497167.159bcfb9a61891ff7e6db58af5cdad8b. 2024-11-16T02:35:01,959 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 159bcfb9a61891ff7e6db58af5cdad8b: Waiting for close lock at 1731724501956Disabling compacts and flushes for region at 1731724501956Disabling writes for close at 1731724501957 (+1 ms)Writing region close event to WAL at 1731724501959 (+2 ms)Closed at 1731724501959 2024-11-16T02:35:01,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741934_1114 (size=175) 2024-11-16T02:35:01,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741934_1114 (size=175) 2024-11-16T02:35:01,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741934_1114 (size=175) 2024-11-16T02:35:01,964 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T02:35:01,964 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731724501754) 2024-11-16T02:35:01,976 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=445 (was 440) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1801539578_22 at /127.0.0.1:49346 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:43879 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1801539578_22 at /127.0.0.1:35834 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1801539578_22 at /127.0.0.1:48536 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:43879 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=298 (was 307), ProcessCount=11 (was 11), AvailableMemoryMB=4070 (was 4086) 2024-11-16T02:35:01,977 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-16T02:35:01,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T02:35:01,977 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T02:35:01,977 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T02:35:01,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T02:35:01,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T02:35:01,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T02:35:01,978 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T02:35:01,978 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=705131862, stopped=false 2024-11-16T02:35:01,978 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=667222f7cf1b,38401,1731724461545 2024-11-16T02:35:01,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T02:35:01,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T02:35:01,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T02:35:01,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:35:01,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:35:01,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:35:01,999 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T02:35:02,000 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T02:35:02,000 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T02:35:02,000 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T02:35:02,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T02:35:02,000 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T02:35:02,000 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T02:35:02,000 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '667222f7cf1b,40547,1731724462323' ***** 2024-11-16T02:35:02,001 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T02:35:02,001 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '667222f7cf1b,39421,1731724462467' ***** 2024-11-16T02:35:02,001 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T02:35:02,001 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T02:35:02,001 INFO [RS:0;667222f7cf1b:40547 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T02:35:02,001 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T02:35:02,001 INFO [RS:0;667222f7cf1b:40547 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T02:35:02,001 INFO [RS:2;667222f7cf1b:39421 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T02:35:02,001 INFO [RS:2;667222f7cf1b:39421 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T02:35:02,001 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(959): stopping server 667222f7cf1b,40547,1731724462323 2024-11-16T02:35:02,001 INFO [RS:0;667222f7cf1b:40547 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T02:35:02,001 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(3091): Received CLOSE for 056b939bd43275b5084f24034a22ab92 2024-11-16T02:35:02,001 INFO [RS:0;667222f7cf1b:40547 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;667222f7cf1b:40547. 2024-11-16T02:35:02,001 DEBUG [RS:0;667222f7cf1b:40547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T02:35:02,001 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T02:35:02,001 DEBUG [RS:0;667222f7cf1b:40547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T02:35:02,001 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T02:35:02,001 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T02:35:02,002 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T02:35:02,002 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(959): stopping server 667222f7cf1b,39421,1731724462467 2024-11-16T02:35:02,002 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T02:35:02,002 INFO [RS:2;667222f7cf1b:39421 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T02:35:02,002 INFO [RS:2;667222f7cf1b:39421 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;667222f7cf1b:39421. 2024-11-16T02:35:02,002 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T02:35:02,002 DEBUG [RS:2;667222f7cf1b:39421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T02:35:02,002 DEBUG [RS:2;667222f7cf1b:39421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T02:35:02,002 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T02:35:02,002 DEBUG [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(1325): Online Regions={056b939bd43275b5084f24034a22ab92=testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92.} 2024-11-16T02:35:02,002 DEBUG [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(1351): Waiting on 056b939bd43275b5084f24034a22ab92 2024-11-16T02:35:02,002 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T02:35:02,002 DEBUG [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T02:35:02,002 DEBUG [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T02:35:02,002 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 056b939bd43275b5084f24034a22ab92, disabling compactions & flushes 2024-11-16T02:35:02,002 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T02:35:02,002 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:35:02,002 INFO [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T02:35:02,002 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:35:02,002 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T02:35:02,002 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. after waiting 0 ms 2024-11-16T02:35:02,002 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T02:35:02,002 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:35:02,002 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T02:35:02,002 INFO [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-16T02:35:02,006 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/056b939bd43275b5084f24034a22ab92/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-16T02:35:02,007 INFO [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:35:02,007 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 056b939bd43275b5084f24034a22ab92: Waiting for close lock at 1731724502002Running coprocessor pre-close hooks at 1731724502002Disabling compacts and flushes for region at 1731724502002Disabling writes for close at 1731724502002Writing region close event to WAL at 1731724502003 (+1 ms)Running coprocessor post-close hooks at 1731724502007 (+4 ms)Closed at 1731724502007 2024-11-16T02:35:02,007 DEBUG [RS_CLOSE_REGION-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92. 2024-11-16T02:35:02,027 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/.tmp/info/6f94994ca11d4b9ebce70da9370b32c1 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731724477751.056b939bd43275b5084f24034a22ab92./info:regioninfo/1731724481002/Put/seqid=0 2024-11-16T02:35:02,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741938_1118 (size=8243) 2024-11-16T02:35:02,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741938_1118 (size=8243) 2024-11-16T02:35:02,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741938_1118 (size=8243) 2024-11-16T02:35:02,034 INFO [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/.tmp/info/6f94994ca11d4b9ebce70da9370b32c1 2024-11-16T02:35:02,040 INFO [regionserver/667222f7cf1b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T02:35:02,040 INFO [regionserver/667222f7cf1b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T02:35:02,053 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/.tmp/ns/a1b54a5539ce4559b5e68cb3d3255d22 is 43, key is default/ns:d/1731724464701/Put/seqid=0 2024-11-16T02:35:02,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741939_1119 (size=5153) 2024-11-16T02:35:02,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741939_1119 (size=5153) 2024-11-16T02:35:02,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741939_1119 (size=5153) 2024-11-16T02:35:02,061 INFO [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/.tmp/ns/a1b54a5539ce4559b5e68cb3d3255d22 2024-11-16T02:35:02,081 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/.tmp/table/ef1f09f31ab54aa881077b817858a530 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731724478191/Put/seqid=0 2024-11-16T02:35:02,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741940_1120 (size=5431) 2024-11-16T02:35:02,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741940_1120 (size=5431) 2024-11-16T02:35:02,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741940_1120 (size=5431) 2024-11-16T02:35:02,088 INFO [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/.tmp/table/ef1f09f31ab54aa881077b817858a530 2024-11-16T02:35:02,093 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/.tmp/info/6f94994ca11d4b9ebce70da9370b32c1 as hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/info/6f94994ca11d4b9ebce70da9370b32c1 2024-11-16T02:35:02,098 INFO [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/info/6f94994ca11d4b9ebce70da9370b32c1, entries=18, sequenceid=21, filesize=8.0 K 2024-11-16T02:35:02,100 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/.tmp/ns/a1b54a5539ce4559b5e68cb3d3255d22 as hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/ns/a1b54a5539ce4559b5e68cb3d3255d22 2024-11-16T02:35:02,105 INFO [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/ns/a1b54a5539ce4559b5e68cb3d3255d22, entries=2, sequenceid=21, filesize=5.0 K 2024-11-16T02:35:02,106 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/.tmp/table/ef1f09f31ab54aa881077b817858a530 as hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/table/ef1f09f31ab54aa881077b817858a530 2024-11-16T02:35:02,112 INFO [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/table/ef1f09f31ab54aa881077b817858a530, entries=2, sequenceid=21, filesize=5.3 K 2024-11-16T02:35:02,113 INFO [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=21, compaction requested=false 2024-11-16T02:35:02,117 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-16T02:35:02,118 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T02:35:02,118 INFO [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T02:35:02,118 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731724502002Running coprocessor pre-close hooks at 1731724502002Disabling compacts and flushes for region at 1731724502002Disabling writes for close at 1731724502002Obtaining lock to block concurrent updates at 1731724502002Preparing flush snapshotting stores in 1588230740 at 1731724502002Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1731724502003 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731724502004 (+1 ms)Flushing 1588230740/info: creating writer at 1731724502004Flushing 1588230740/info: appending metadata at 1731724502027 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731724502027Flushing 1588230740/ns: creating writer at 1731724502040 (+13 ms)Flushing 1588230740/ns: appending metadata at 1731724502053 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731724502053Flushing 1588230740/table: creating writer at 1731724502067 (+14 ms)Flushing 1588230740/table: appending metadata at 1731724502081 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731724502081Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4462d0c1: reopening flushed file at 1731724502093 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57471100: reopening flushed file at 1731724502099 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e16a215: reopening flushed file at 1731724502105 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=21, compaction requested=false at 1731724502113 (+8 ms)Writing region close event to WAL at 1731724502114 (+1 ms)Running coprocessor post-close hooks at 1731724502118 (+4 ms)Closed at 1731724502118 2024-11-16T02:35:02,119 DEBUG [RS_CLOSE_META-regionserver/667222f7cf1b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T02:35:02,202 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(976): stopping server 667222f7cf1b,39421,1731724462467; all regions closed. 2024-11-16T02:35:02,202 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(976): stopping server 667222f7cf1b,40547,1731724462323; all regions closed. 2024-11-16T02:35:02,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741836_1012 (size=9487) 2024-11-16T02:35:02,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741835_1011 (size=1032) 2024-11-16T02:35:02,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741835_1011 (size=1032) 2024-11-16T02:35:02,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741836_1012 (size=9487) 2024-11-16T02:35:02,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741835_1011 (size=1032) 2024-11-16T02:35:02,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741836_1012 (size=9487) 2024-11-16T02:35:02,212 DEBUG [RS:2;667222f7cf1b:39421 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/oldWALs 2024-11-16T02:35:02,212 INFO [RS:2;667222f7cf1b:39421 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 667222f7cf1b%2C39421%2C1731724462467:(num 1731724464057) 2024-11-16T02:35:02,212 DEBUG [RS:2;667222f7cf1b:39421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T02:35:02,212 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T02:35:02,212 INFO [RS:2;667222f7cf1b:39421 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T02:35:02,212 DEBUG [RS:0;667222f7cf1b:40547 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/oldWALs 2024-11-16T02:35:02,212 INFO [RS:0;667222f7cf1b:40547 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 667222f7cf1b%2C40547%2C1731724462323.meta:.meta(num 1731724464462) 2024-11-16T02:35:02,212 INFO [RS:2;667222f7cf1b:39421 {}] hbase.ChoreService(370): Chore service for: regionserver/667222f7cf1b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T02:35:02,212 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T02:35:02,212 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T02:35:02,212 INFO [regionserver/667222f7cf1b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T02:35:02,212 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T02:35:02,213 INFO [RS:2;667222f7cf1b:39421 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T02:35:02,213 INFO [RS:2;667222f7cf1b:39421 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39421 2024-11-16T02:35:02,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741833_1009 (size=2764) 2024-11-16T02:35:02,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741833_1009 (size=2764) 2024-11-16T02:35:02,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741833_1009 (size=2764) 2024-11-16T02:35:02,217 DEBUG [RS:0;667222f7cf1b:40547 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/oldWALs 2024-11-16T02:35:02,217 INFO [RS:0;667222f7cf1b:40547 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 667222f7cf1b%2C40547%2C1731724462323:(num 1731724464057) 2024-11-16T02:35:02,217 DEBUG [RS:0;667222f7cf1b:40547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T02:35:02,217 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T02:35:02,217 INFO [RS:0;667222f7cf1b:40547 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T02:35:02,217 INFO [RS:0;667222f7cf1b:40547 {}] hbase.ChoreService(370): Chore service for: regionserver/667222f7cf1b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T02:35:02,217 INFO [RS:0;667222f7cf1b:40547 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T02:35:02,217 INFO [regionserver/667222f7cf1b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T02:35:02,217 INFO [RS:0;667222f7cf1b:40547 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40547 2024-11-16T02:35:02,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T02:35:02,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/667222f7cf1b,39421,1731724462467 2024-11-16T02:35:02,220 INFO [RS:2;667222f7cf1b:39421 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T02:35:02,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/667222f7cf1b,40547,1731724462323 2024-11-16T02:35:02,230 INFO [RS:0;667222f7cf1b:40547 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T02:35:02,241 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [667222f7cf1b,39421,1731724462467] 2024-11-16T02:35:02,262 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/667222f7cf1b,39421,1731724462467 already deleted, retry=false 2024-11-16T02:35:02,262 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 667222f7cf1b,39421,1731724462467 expired; onlineServers=1 2024-11-16T02:35:02,262 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [667222f7cf1b,40547,1731724462323] 2024-11-16T02:35:02,273 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/667222f7cf1b,40547,1731724462323 already deleted, retry=false 2024-11-16T02:35:02,273 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 667222f7cf1b,40547,1731724462323 expired; onlineServers=0 2024-11-16T02:35:02,273 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '667222f7cf1b,38401,1731724461545' ***** 2024-11-16T02:35:02,273 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T02:35:02,273 INFO [M:0;667222f7cf1b:38401 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T02:35:02,273 INFO [M:0;667222f7cf1b:38401 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T02:35:02,273 DEBUG [M:0;667222f7cf1b:38401 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T02:35:02,273 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T02:35:02,273 DEBUG [M:0;667222f7cf1b:38401 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T02:35:02,273 DEBUG [master/667222f7cf1b:0:becomeActiveMaster-HFileCleaner.large.0-1731724463645 {}] cleaner.HFileCleaner(306): Exit Thread[master/667222f7cf1b:0:becomeActiveMaster-HFileCleaner.large.0-1731724463645,5,FailOnTimeoutGroup] 2024-11-16T02:35:02,273 DEBUG [master/667222f7cf1b:0:becomeActiveMaster-HFileCleaner.small.0-1731724463651 {}] cleaner.HFileCleaner(306): Exit Thread[master/667222f7cf1b:0:becomeActiveMaster-HFileCleaner.small.0-1731724463651,5,FailOnTimeoutGroup] 2024-11-16T02:35:02,274 INFO [M:0;667222f7cf1b:38401 {}] hbase.ChoreService(370): Chore service for: master/667222f7cf1b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T02:35:02,274 INFO [M:0;667222f7cf1b:38401 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T02:35:02,274 DEBUG [M:0;667222f7cf1b:38401 {}] master.HMaster(1795): Stopping service threads 2024-11-16T02:35:02,274 INFO [M:0;667222f7cf1b:38401 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T02:35:02,274 INFO [M:0;667222f7cf1b:38401 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T02:35:02,275 INFO [M:0;667222f7cf1b:38401 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T02:35:02,275 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T02:35:02,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T02:35:02,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T02:35:02,284 DEBUG [M:0;667222f7cf1b:38401 {}] zookeeper.ZKUtil(347): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T02:35:02,284 WARN [M:0;667222f7cf1b:38401 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T02:35:02,285 INFO [M:0;667222f7cf1b:38401 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/.lastflushedseqids 2024-11-16T02:35:02,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741941_1121 (size=138) 2024-11-16T02:35:02,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741941_1121 (size=138) 2024-11-16T02:35:02,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741941_1121 (size=138) 2024-11-16T02:35:02,303 INFO [M:0;667222f7cf1b:38401 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T02:35:02,303 INFO [M:0;667222f7cf1b:38401 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T02:35:02,303 DEBUG [M:0;667222f7cf1b:38401 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T02:35:02,303 INFO [M:0;667222f7cf1b:38401 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T02:35:02,303 DEBUG [M:0;667222f7cf1b:38401 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T02:35:02,303 DEBUG [M:0;667222f7cf1b:38401 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T02:35:02,303 DEBUG [M:0;667222f7cf1b:38401 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T02:35:02,303 INFO [M:0;667222f7cf1b:38401 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-16T02:35:02,318 DEBUG [M:0;667222f7cf1b:38401 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f6f89555ae654b7987eceaac6de387c9 is 82, key is hbase:meta,,1/info:regioninfo/1731724464576/Put/seqid=0 2024-11-16T02:35:02,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741942_1122 (size=5672) 2024-11-16T02:35:02,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741942_1122 (size=5672) 2024-11-16T02:35:02,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741942_1122 (size=5672) 2024-11-16T02:35:02,324 INFO [M:0;667222f7cf1b:38401 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f6f89555ae654b7987eceaac6de387c9 2024-11-16T02:35:02,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T02:35:02,341 INFO [RS:2;667222f7cf1b:39421 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T02:35:02,341 INFO [RS:2;667222f7cf1b:39421 {}] regionserver.HRegionServer(1031): Exiting; stopping=667222f7cf1b,39421,1731724462467; zookeeper connection closed. 2024-11-16T02:35:02,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39421-0x101417e3e890003, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T02:35:02,341 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@51acd5b1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@51acd5b1 2024-11-16T02:35:02,346 DEBUG [M:0;667222f7cf1b:38401 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ca6930556784926afba8cf121c1a21c is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731724478199/Put/seqid=0 2024-11-16T02:35:02,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T02:35:02,352 INFO [RS:0;667222f7cf1b:40547 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T02:35:02,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40547-0x101417e3e890001, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T02:35:02,352 INFO [RS:0;667222f7cf1b:40547 {}] regionserver.HRegionServer(1031): Exiting; stopping=667222f7cf1b,40547,1731724462323; zookeeper connection closed. 2024-11-16T02:35:02,352 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@364abf85 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@364abf85 2024-11-16T02:35:02,353 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-16T02:35:02,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741943_1123 (size=7755) 2024-11-16T02:35:02,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741943_1123 (size=7755) 2024-11-16T02:35:02,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741943_1123 (size=7755) 2024-11-16T02:35:02,355 INFO [M:0;667222f7cf1b:38401 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ca6930556784926afba8cf121c1a21c 2024-11-16T02:35:02,358 INFO [M:0;667222f7cf1b:38401 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7ca6930556784926afba8cf121c1a21c 2024-11-16T02:35:02,372 DEBUG [M:0;667222f7cf1b:38401 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/801dc6b197d94358b87fe4b81093b646 is 69, key is 667222f7cf1b,39421,1731724462467/rs:state/1731724463767/Put/seqid=0 2024-11-16T02:35:02,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741944_1124 (size=5445) 2024-11-16T02:35:02,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741944_1124 (size=5445) 2024-11-16T02:35:02,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741944_1124 (size=5445) 2024-11-16T02:35:02,378 INFO [M:0;667222f7cf1b:38401 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/801dc6b197d94358b87fe4b81093b646 2024-11-16T02:35:02,381 INFO [M:0;667222f7cf1b:38401 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 801dc6b197d94358b87fe4b81093b646 2024-11-16T02:35:02,382 DEBUG [M:0;667222f7cf1b:38401 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f6f89555ae654b7987eceaac6de387c9 as hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f6f89555ae654b7987eceaac6de387c9 2024-11-16T02:35:02,387 INFO [M:0;667222f7cf1b:38401 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f6f89555ae654b7987eceaac6de387c9, entries=8, sequenceid=168, filesize=5.5 K 2024-11-16T02:35:02,388 DEBUG [M:0;667222f7cf1b:38401 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ca6930556784926afba8cf121c1a21c as hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7ca6930556784926afba8cf121c1a21c 2024-11-16T02:35:02,392 INFO [M:0;667222f7cf1b:38401 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7ca6930556784926afba8cf121c1a21c 2024-11-16T02:35:02,392 INFO [M:0;667222f7cf1b:38401 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7ca6930556784926afba8cf121c1a21c, entries=17, sequenceid=168, filesize=7.6 K 2024-11-16T02:35:02,393 DEBUG [M:0;667222f7cf1b:38401 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/801dc6b197d94358b87fe4b81093b646 as hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/801dc6b197d94358b87fe4b81093b646 2024-11-16T02:35:02,398 INFO [M:0;667222f7cf1b:38401 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 801dc6b197d94358b87fe4b81093b646 2024-11-16T02:35:02,398 INFO [M:0;667222f7cf1b:38401 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43879/user/jenkins/test-data/394987c6-7d0a-3d90-8bd7-439595e1d7b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/801dc6b197d94358b87fe4b81093b646, entries=3, sequenceid=168, filesize=5.3 K 2024-11-16T02:35:02,399 INFO [M:0;667222f7cf1b:38401 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 96ms, sequenceid=168, compaction requested=false 2024-11-16T02:35:02,402 INFO [M:0;667222f7cf1b:38401 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T02:35:02,402 DEBUG [M:0;667222f7cf1b:38401 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731724502303Disabling compacts and flushes for region at 1731724502303Disabling writes for close at 1731724502303Obtaining lock to block concurrent updates at 1731724502303Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731724502303Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1731724502303Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731724502304 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731724502304Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731724502318 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731724502318Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731724502328 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731724502345 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731724502345Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731724502359 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731724502371 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731724502371Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4199481b: reopening flushed file at 1731724502381 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12b1ab7: reopening flushed file at 1731724502387 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29c3f17b: reopening flushed file at 1731724502393 (+6 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 96ms, sequenceid=168, compaction requested=false at 1731724502399 (+6 ms)Writing region close event to WAL at 1731724502401 (+2 ms)Closed at 1731724502401 2024-11-16T02:35:02,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39531 is added to blk_1073741830_1006 (size=83606) 2024-11-16T02:35:02,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40973 is added to blk_1073741830_1006 (size=83606) 2024-11-16T02:35:02,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33993 is added to blk_1073741830_1006 (size=83606) 2024-11-16T02:35:02,405 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T02:35:02,405 INFO [M:0;667222f7cf1b:38401 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T02:35:02,405 INFO [M:0;667222f7cf1b:38401 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38401 2024-11-16T02:35:02,406 INFO [M:0;667222f7cf1b:38401 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T02:35:02,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T02:35:02,517 INFO [M:0;667222f7cf1b:38401 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T02:35:02,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38401-0x101417e3e890000, quorum=127.0.0.1:51370, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T02:35:02,529 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481655 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481655 (inode 16655) Holder DFSClient_NONMAPREDUCE_464715242_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731724481258/wal.1731724481655 (inode 16655) Holder DFSClient_NONMAPREDUCE_464715242_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-16T02:35:02,530 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731724473213/wal.1731724473344 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:35:02,531 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731724497166/wal.1731724497432 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:35:02,533 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724491182 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724491182 (inode 16678) Holder DFSClient_NONMAPREDUCE_464715242_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731724481831/wal.1731724491182 (inode 16678) Holder DFSClient_NONMAPREDUCE_464715242_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-16T02:35:02,533 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731724466350/wal.1731724466442 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:35:02,535 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724492027 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724492027 (inode 16704) Holder DFSClient_NONMAPREDUCE_464715242_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731724491416/wal.1731724492027 (inode 16704) Holder DFSClient_NONMAPREDUCE_464715242_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-16T02:35:02,536 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731724492219/wal.1731724492300 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T02:35:02,538 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal.1731724466174 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal.1731724466174 (inode 16485) Holder DFSClient_NONMAPREDUCE_464715242_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731724465815/wal.1731724466174 (inode 16485) Holder DFSClient_NONMAPREDUCE_464715242_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-16T02:35:02,540 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal.1731724465537 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal.1731724465537 (inode 16462) Holder DFSClient_NONMAPREDUCE_464715242_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731724464991/wal.1731724465537 (inode 16462) Holder DFSClient_NONMAPREDUCE_464715242_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-16T02:35:02,544 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T02:35:02,546 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T02:35:02,546 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T02:35:02,546 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T02:35:02,546 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/hadoop.log.dir/,STOPPED} 2024-11-16T02:35:02,548 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T02:35:02,548 WARN [BP-1221085024-172.17.0.3-1731724457043 heartbeating to localhost/127.0.0.1:43879 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T02:35:02,548 WARN [BP-1221085024-172.17.0.3-1731724457043 heartbeating to localhost/127.0.0.1:43879 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1221085024-172.17.0.3-1731724457043 (Datanode Uuid 39717887-3138-4201-8842-0eece2fe6967) service to localhost/127.0.0.1:43879 2024-11-16T02:35:02,548 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T02:35:02,550 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data5/current/BP-1221085024-172.17.0.3-1731724457043 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T02:35:02,550 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data6/current/BP-1221085024-172.17.0.3-1731724457043 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T02:35:02,551 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T02:35:02,552 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T02:35:02,553 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T02:35:02,553 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T02:35:02,553 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T02:35:02,553 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/hadoop.log.dir/,STOPPED} 2024-11-16T02:35:02,554 WARN [BP-1221085024-172.17.0.3-1731724457043 heartbeating to localhost/127.0.0.1:43879 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T02:35:02,554 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T02:35:02,554 WARN [BP-1221085024-172.17.0.3-1731724457043 heartbeating to localhost/127.0.0.1:43879 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1221085024-172.17.0.3-1731724457043 (Datanode Uuid 3f36a101-aafe-43cc-8014-0d1f212b982f) service to localhost/127.0.0.1:43879 2024-11-16T02:35:02,554 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T02:35:02,555 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data3/current/BP-1221085024-172.17.0.3-1731724457043 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T02:35:02,555 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data4/current/BP-1221085024-172.17.0.3-1731724457043 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T02:35:02,555 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T02:35:02,557 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T02:35:02,557 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T02:35:02,557 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T02:35:02,557 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T02:35:02,557 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/hadoop.log.dir/,STOPPED} 2024-11-16T02:35:02,558 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T02:35:02,558 WARN [BP-1221085024-172.17.0.3-1731724457043 heartbeating to localhost/127.0.0.1:43879 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T02:35:02,558 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T02:35:02,558 WARN [BP-1221085024-172.17.0.3-1731724457043 heartbeating to localhost/127.0.0.1:43879 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1221085024-172.17.0.3-1731724457043 (Datanode Uuid ba902d54-5219-4efc-bd09-dd1188434d48) service to localhost/127.0.0.1:43879 2024-11-16T02:35:02,559 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data1/current/BP-1221085024-172.17.0.3-1731724457043 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T02:35:02,559 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/cluster_38dec838-5959-ab14-96fd-104acdff8d39/data/data2/current/BP-1221085024-172.17.0.3-1731724457043 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T02:35:02,560 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T02:35:02,572 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T02:35:02,573 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T02:35:02,573 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T02:35:02,574 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T02:35:02,574 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44f0e2a4-cb13-15c7-6648-db664a9b47d2/hadoop.log.dir/,STOPPED} 2024-11-16T02:35:02,582 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T02:35:02,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down