2024-12-01 18:49:36,873 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-01 18:49:36,883 main DEBUG Took 0.008767 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-01 18:49:36,884 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-01 18:49:36,884 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-01 18:49:36,885 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-01 18:49:36,886 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,892 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-01 18:49:36,904 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,905 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,905 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,906 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,906 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,906 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,907 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,908 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,908 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,908 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,909 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,909 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,910 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,910 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,911 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,912 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,912 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,913 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,913 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,913 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,914 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 18:49:36,914 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,914 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-01 18:49:36,915 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 18:49:36,917 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-01 18:49:36,918 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-01 18:49:36,919 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-01 18:49:36,920 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-01 18:49:36,920 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-01 18:49:36,928 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-01 18:49:36,930 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-01 18:49:36,931 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-01 18:49:36,932 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-01 18:49:36,932 main DEBUG createAppenders(={Console}) 2024-12-01 18:49:36,933 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-01 18:49:36,933 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-01 18:49:36,933 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-01 18:49:36,934 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-01 18:49:36,934 main DEBUG OutputStream closed 2024-12-01 18:49:36,934 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-01 18:49:36,934 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-01 18:49:36,935 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-01 18:49:37,000 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-01 18:49:37,002 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-01 18:49:37,003 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-01 18:49:37,004 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-01 18:49:37,004 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-01 18:49:37,004 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-01 18:49:37,005 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-01 18:49:37,005 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-01 18:49:37,005 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-01 18:49:37,006 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-01 18:49:37,006 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-01 18:49:37,006 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-01 18:49:37,006 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-01 18:49:37,007 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-01 18:49:37,007 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-01 18:49:37,007 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-01 18:49:37,007 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-01 18:49:37,008 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-01 18:49:37,010 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01 18:49:37,011 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-01 18:49:37,011 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-01 18:49:37,012 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-01T18:49:37,230 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff 2024-12-01 18:49:37,233 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-01 18:49:37,233 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01T18:49:37,242 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-01T18:49:37,248 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-01T18:49:37,268 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-01T18:49:37,313 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-01T18:49:37,313 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-01T18:49:37,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-01T18:49:37,346 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c, deleteOnExit=true 2024-12-01T18:49:37,347 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-01T18:49:37,348 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/test.cache.data in system properties and HBase conf 2024-12-01T18:49:37,348 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/hadoop.tmp.dir in system properties and HBase conf 2024-12-01T18:49:37,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/hadoop.log.dir in system properties and HBase conf 2024-12-01T18:49:37,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-01T18:49:37,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-01T18:49:37,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-01T18:49:37,435 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-01T18:49:37,530 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T18:49:37,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-01T18:49:37,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-01T18:49:37,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-01T18:49:37,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-01T18:49:37,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-01T18:49:37,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-01T18:49:37,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-01T18:49:37,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-01T18:49:37,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-01T18:49:37,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/nfs.dump.dir in system properties and HBase conf 2024-12-01T18:49:37,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/java.io.tmpdir in system properties and HBase conf 2024-12-01T18:49:37,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-01T18:49:37,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-01T18:49:37,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-01T18:49:38,337 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-01T18:49:38,417 INFO [Time-limited test {}] log.Log(170): Logging initialized @2304ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-01T18:49:38,492 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T18:49:38,555 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T18:49:38,583 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T18:49:38,583 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T18:49:38,585 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-01T18:49:38,597 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T18:49:38,599 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/hadoop.log.dir/,AVAILABLE} 2024-12-01T18:49:38,600 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T18:49:38,790 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/java.io.tmpdir/jetty-localhost-39103-hadoop-hdfs-3_4_1-tests_jar-_-any-1933489788150562563/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-01T18:49:38,796 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:39103} 2024-12-01T18:49:38,797 INFO [Time-limited test {}] server.Server(415): Started @2683ms 2024-12-01T18:49:39,181 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T18:49:39,188 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T18:49:39,189 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T18:49:39,189 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T18:49:39,189 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T18:49:39,190 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/hadoop.log.dir/,AVAILABLE} 2024-12-01T18:49:39,191 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T18:49:39,312 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/java.io.tmpdir/jetty-localhost-36475-hadoop-hdfs-3_4_1-tests_jar-_-any-12625227295181970291/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T18:49:39,313 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:36475} 2024-12-01T18:49:39,313 INFO [Time-limited test {}] server.Server(415): Started @3199ms 2024-12-01T18:49:39,365 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-01T18:49:39,502 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T18:49:39,510 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T18:49:39,519 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T18:49:39,519 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T18:49:39,520 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T18:49:39,520 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/hadoop.log.dir/,AVAILABLE} 2024-12-01T18:49:39,521 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T18:49:39,668 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/java.io.tmpdir/jetty-localhost-42735-hadoop-hdfs-3_4_1-tests_jar-_-any-13460576402528576247/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T18:49:39,669 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:42735} 2024-12-01T18:49:39,670 INFO [Time-limited test {}] server.Server(415): Started @3556ms 2024-12-01T18:49:39,673 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-01T18:49:39,720 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T18:49:39,726 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T18:49:39,731 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T18:49:39,731 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T18:49:39,731 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T18:49:39,733 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/hadoop.log.dir/,AVAILABLE} 2024-12-01T18:49:39,734 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T18:49:39,866 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/java.io.tmpdir/jetty-localhost-43415-hadoop-hdfs-3_4_1-tests_jar-_-any-8993751044215373221/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T18:49:39,866 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:43415} 2024-12-01T18:49:39,867 INFO [Time-limited test {}] server.Server(415): Started @3753ms 2024-12-01T18:49:39,869 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-01T18:49:39,873 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data3/current/BP-1540968591-172.17.0.2-1733078978096/current, will proceed with Du for space computation calculation, 2024-12-01T18:49:39,873 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data2/current/BP-1540968591-172.17.0.2-1733078978096/current, will proceed with Du for space computation calculation, 2024-12-01T18:49:39,873 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data1/current/BP-1540968591-172.17.0.2-1733078978096/current, will proceed with Du for space computation calculation, 2024-12-01T18:49:39,873 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data4/current/BP-1540968591-172.17.0.2-1733078978096/current, will proceed with Du for space computation calculation, 2024-12-01T18:49:39,940 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-01T18:49:39,940 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-01T18:49:40,014 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data5/current/BP-1540968591-172.17.0.2-1733078978096/current, will proceed with Du for space computation calculation, 2024-12-01T18:49:40,015 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data6/current/BP-1540968591-172.17.0.2-1733078978096/current, will proceed with Du for space computation calculation, 2024-12-01T18:49:40,024 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6cd4c024d21828ef with lease ID 0xc16de8714cf67654: Processing first storage report for DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f from datanode DatanodeRegistration(127.0.0.1:35899, datanodeUuid=06542ef0-158e-42de-b8da-c2e2b0c303cd, infoPort=33601, infoSecurePort=0, ipcPort=36499, storageInfo=lv=-57;cid=testClusterID;nsid=1208402048;c=1733078978096) 2024-12-01T18:49:40,026 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6cd4c024d21828ef with lease ID 0xc16de8714cf67654: from storage DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f node DatanodeRegistration(127.0.0.1:35899, datanodeUuid=06542ef0-158e-42de-b8da-c2e2b0c303cd, infoPort=33601, infoSecurePort=0, ipcPort=36499, storageInfo=lv=-57;cid=testClusterID;nsid=1208402048;c=1733078978096), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-01T18:49:40,026 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x925c1c28ce3bf264 with lease ID 0xc16de8714cf67655: Processing first storage report for DS-4811a14f-9cd8-41a9-be38-25d01bc29430 from datanode DatanodeRegistration(127.0.0.1:41037, datanodeUuid=da0abfe6-180d-4c93-825e-ed7b0c31716a, infoPort=37787, infoSecurePort=0, ipcPort=43397, storageInfo=lv=-57;cid=testClusterID;nsid=1208402048;c=1733078978096) 2024-12-01T18:49:40,026 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x925c1c28ce3bf264 with lease ID 0xc16de8714cf67655: from storage DS-4811a14f-9cd8-41a9-be38-25d01bc29430 node DatanodeRegistration(127.0.0.1:41037, datanodeUuid=da0abfe6-180d-4c93-825e-ed7b0c31716a, infoPort=37787, infoSecurePort=0, ipcPort=43397, storageInfo=lv=-57;cid=testClusterID;nsid=1208402048;c=1733078978096), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T18:49:40,026 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6cd4c024d21828ef with lease ID 0xc16de8714cf67654: Processing first storage report for DS-78c95881-65ab-4e7d-abb1-398976bb5820 from datanode DatanodeRegistration(127.0.0.1:35899, datanodeUuid=06542ef0-158e-42de-b8da-c2e2b0c303cd, infoPort=33601, infoSecurePort=0, ipcPort=36499, storageInfo=lv=-57;cid=testClusterID;nsid=1208402048;c=1733078978096) 2024-12-01T18:49:40,027 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6cd4c024d21828ef with lease ID 0xc16de8714cf67654: from storage DS-78c95881-65ab-4e7d-abb1-398976bb5820 node DatanodeRegistration(127.0.0.1:35899, datanodeUuid=06542ef0-158e-42de-b8da-c2e2b0c303cd, infoPort=33601, infoSecurePort=0, ipcPort=36499, storageInfo=lv=-57;cid=testClusterID;nsid=1208402048;c=1733078978096), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T18:49:40,027 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x925c1c28ce3bf264 with lease ID 0xc16de8714cf67655: Processing first storage report for DS-e693363d-84e4-4e83-9414-a25379a2670e from datanode DatanodeRegistration(127.0.0.1:41037, datanodeUuid=da0abfe6-180d-4c93-825e-ed7b0c31716a, infoPort=37787, infoSecurePort=0, ipcPort=43397, storageInfo=lv=-57;cid=testClusterID;nsid=1208402048;c=1733078978096) 2024-12-01T18:49:40,027 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x925c1c28ce3bf264 with lease ID 0xc16de8714cf67655: from storage DS-e693363d-84e4-4e83-9414-a25379a2670e node DatanodeRegistration(127.0.0.1:41037, datanodeUuid=da0abfe6-180d-4c93-825e-ed7b0c31716a, infoPort=37787, infoSecurePort=0, ipcPort=43397, storageInfo=lv=-57;cid=testClusterID;nsid=1208402048;c=1733078978096), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T18:49:40,054 WARN [Thread-119 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-01T18:49:40,061 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x79b09791c3dc15b8 with lease ID 0xc16de8714cf67656: Processing first storage report for DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98 from datanode DatanodeRegistration(127.0.0.1:43787, datanodeUuid=a1e2453e-a549-4712-98fd-e539af60652c, infoPort=40921, infoSecurePort=0, ipcPort=33503, storageInfo=lv=-57;cid=testClusterID;nsid=1208402048;c=1733078978096) 2024-12-01T18:49:40,061 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x79b09791c3dc15b8 with lease ID 0xc16de8714cf67656: from storage DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98 node DatanodeRegistration(127.0.0.1:43787, datanodeUuid=a1e2453e-a549-4712-98fd-e539af60652c, infoPort=40921, infoSecurePort=0, ipcPort=33503, storageInfo=lv=-57;cid=testClusterID;nsid=1208402048;c=1733078978096), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T18:49:40,061 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x79b09791c3dc15b8 with lease ID 0xc16de8714cf67656: Processing first storage report for DS-0ccf9431-054a-4e0b-98f4-77754182b21b from datanode DatanodeRegistration(127.0.0.1:43787, datanodeUuid=a1e2453e-a549-4712-98fd-e539af60652c, infoPort=40921, infoSecurePort=0, ipcPort=33503, storageInfo=lv=-57;cid=testClusterID;nsid=1208402048;c=1733078978096) 2024-12-01T18:49:40,061 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x79b09791c3dc15b8 with lease ID 0xc16de8714cf67656: from storage DS-0ccf9431-054a-4e0b-98f4-77754182b21b node DatanodeRegistration(127.0.0.1:43787, datanodeUuid=a1e2453e-a549-4712-98fd-e539af60652c, infoPort=40921, infoSecurePort=0, ipcPort=33503, storageInfo=lv=-57;cid=testClusterID;nsid=1208402048;c=1733078978096), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T18:49:40,272 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff 2024-12-01T18:49:40,343 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/zookeeper_0, clientPort=55741, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-01T18:49:40,352 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55741 2024-12-01T18:49:40,362 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T18:49:40,364 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T18:49:40,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741825_1001 (size=7) 2024-12-01T18:49:40,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741825_1001 (size=7) 2024-12-01T18:49:40,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741825_1001 (size=7) 2024-12-01T18:49:41,001 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018 with version=8 2024-12-01T18:49:41,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/hbase-staging 2024-12-01T18:49:41,309 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9ec37ae3355e:0 server-side Connection retries=45 2024-12-01T18:49:41,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T18:49:41,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T18:49:41,324 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T18:49:41,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T18:49:41,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T18:49:41,463 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-01T18:49:41,525 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-01T18:49:41,535 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-01T18:49:41,539 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T18:49:41,568 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 31474 (auto-detected) 2024-12-01T18:49:41,569 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-01T18:49:41,589 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40221 2024-12-01T18:49:41,613 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40221 connecting to ZooKeeper ensemble=127.0.0.1:55741 2024-12-01T18:49:41,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:402210x0, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T18:49:41,646 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40221-0x1016f5c81760000 connected 2024-12-01T18:49:41,675 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T18:49:41,679 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T18:49:41,691 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T18:49:41,695 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018, hbase.cluster.distributed=false 2024-12-01T18:49:41,719 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T18:49:41,723 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40221 2024-12-01T18:49:41,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40221 2024-12-01T18:49:41,725 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40221 2024-12-01T18:49:41,727 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40221 2024-12-01T18:49:41,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40221 2024-12-01T18:49:41,847 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9ec37ae3355e:0 server-side Connection retries=45 2024-12-01T18:49:41,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T18:49:41,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T18:49:41,849 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T18:49:41,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T18:49:41,850 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T18:49:41,852 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T18:49:41,855 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T18:49:41,856 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37405 2024-12-01T18:49:41,858 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37405 connecting to ZooKeeper ensemble=127.0.0.1:55741 2024-12-01T18:49:41,859 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T18:49:41,863 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T18:49:41,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374050x0, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T18:49:41,871 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37405-0x1016f5c81760001 connected 2024-12-01T18:49:41,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T18:49:41,877 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T18:49:41,888 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T18:49:41,890 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T18:49:41,895 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T18:49:41,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37405 2024-12-01T18:49:41,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37405 2024-12-01T18:49:41,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37405 2024-12-01T18:49:41,897 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37405 2024-12-01T18:49:41,897 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37405 2024-12-01T18:49:41,915 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9ec37ae3355e:0 server-side Connection retries=45 2024-12-01T18:49:41,915 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T18:49:41,915 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T18:49:41,916 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T18:49:41,916 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T18:49:41,916 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T18:49:41,916 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T18:49:41,917 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T18:49:41,918 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35069 2024-12-01T18:49:41,919 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35069 connecting to ZooKeeper ensemble=127.0.0.1:55741 2024-12-01T18:49:41,920 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T18:49:41,922 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T18:49:41,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:350690x0, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T18:49:41,929 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T18:49:41,929 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35069-0x1016f5c81760002 connected 2024-12-01T18:49:41,930 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T18:49:41,931 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T18:49:41,932 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T18:49:41,934 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T18:49:41,934 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35069 2024-12-01T18:49:41,935 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35069 2024-12-01T18:49:41,935 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35069 2024-12-01T18:49:41,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35069 2024-12-01T18:49:41,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35069 2024-12-01T18:49:41,953 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9ec37ae3355e:0 server-side Connection retries=45 2024-12-01T18:49:41,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T18:49:41,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T18:49:41,953 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T18:49:41,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T18:49:41,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T18:49:41,953 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T18:49:41,954 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T18:49:41,955 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34539 2024-12-01T18:49:41,956 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34539 connecting to ZooKeeper ensemble=127.0.0.1:55741 2024-12-01T18:49:41,957 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T18:49:41,960 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T18:49:41,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:345390x0, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T18:49:41,966 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:345390x0, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T18:49:41,966 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34539-0x1016f5c81760003 connected 2024-12-01T18:49:41,967 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T18:49:41,967 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T18:49:41,969 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T18:49:41,970 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T18:49:41,972 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34539 2024-12-01T18:49:41,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34539 2024-12-01T18:49:41,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34539 2024-12-01T18:49:41,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34539 2024-12-01T18:49:41,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34539 2024-12-01T18:49:41,992 DEBUG [M:0;9ec37ae3355e:40221 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9ec37ae3355e:40221 2024-12-01T18:49:41,993 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9ec37ae3355e,40221,1733078981110 2024-12-01T18:49:42,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T18:49:42,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T18:49:42,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T18:49:42,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T18:49:42,004 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9ec37ae3355e,40221,1733078981110 2024-12-01T18:49:42,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T18:49:42,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T18:49:42,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T18:49:42,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,026 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-01T18:49:42,027 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9ec37ae3355e,40221,1733078981110 from backup master directory 2024-12-01T18:49:42,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T18:49:42,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9ec37ae3355e,40221,1733078981110 2024-12-01T18:49:42,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T18:49:42,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T18:49:42,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T18:49:42,034 WARN [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T18:49:42,034 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9ec37ae3355e,40221,1733078981110 2024-12-01T18:49:42,037 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-01T18:49:42,038 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-01T18:49:42,092 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/hbase.id] with ID: bd982dac-d7cc-4a21-a476-59f23a0409a3 2024-12-01T18:49:42,093 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/.tmp/hbase.id 2024-12-01T18:49:42,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741826_1002 (size=42) 2024-12-01T18:49:42,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741826_1002 (size=42) 2024-12-01T18:49:42,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741826_1002 (size=42) 2024-12-01T18:49:42,107 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/.tmp/hbase.id]:[hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/hbase.id] 2024-12-01T18:49:42,153 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T18:49:42,157 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-01T18:49:42,176 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-12-01T18:49:42,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741827_1003 (size=196) 2024-12-01T18:49:42,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741827_1003 (size=196) 2024-12-01T18:49:42,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741827_1003 (size=196) 2024-12-01T18:49:42,215 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T18:49:42,216 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-01T18:49:42,222 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:42,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741828_1004 (size=1189) 2024-12-01T18:49:42,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741828_1004 (size=1189) 2024-12-01T18:49:42,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741828_1004 (size=1189) 2024-12-01T18:49:42,282 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store 2024-12-01T18:49:42,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741829_1005 (size=34) 2024-12-01T18:49:42,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741829_1005 (size=34) 2024-12-01T18:49:42,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741829_1005 (size=34) 2024-12-01T18:49:42,310 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-01T18:49:42,313 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:42,314 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T18:49:42,315 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T18:49:42,315 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T18:49:42,316 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T18:49:42,316 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T18:49:42,316 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T18:49:42,317 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733078982314Disabling compacts and flushes for region at 1733078982314Disabling writes for close at 1733078982316 (+2 ms)Writing region close event to WAL at 1733078982316Closed at 1733078982316 2024-12-01T18:49:42,319 WARN [master/9ec37ae3355e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/.initializing 2024-12-01T18:49:42,319 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/WALs/9ec37ae3355e,40221,1733078981110 2024-12-01T18:49:42,327 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:49:42,342 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ec37ae3355e%2C40221%2C1733078981110, suffix=, logDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/WALs/9ec37ae3355e,40221,1733078981110, archiveDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/oldWALs, maxLogs=10 2024-12-01T18:49:42,371 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/WALs/9ec37ae3355e,40221,1733078981110/9ec37ae3355e%2C40221%2C1733078981110.1733078982346, exclude list is [], retry=0 2024-12-01T18:49:42,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:49:42,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:42,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:42,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:42,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-01T18:49:42,435 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-01T18:49:42,500 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/WALs/9ec37ae3355e,40221,1733078981110/9ec37ae3355e%2C40221%2C1733078981110.1733078982346 2024-12-01T18:49:42,501 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921)] 2024-12-01T18:49:42,502 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:42,502 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:42,506 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T18:49:42,506 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T18:49:42,543 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T18:49:42,570 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-01T18:49:42,574 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:42,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:49:42,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T18:49:42,580 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-01T18:49:42,580 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:42,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:42,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T18:49:42,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-01T18:49:42,584 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:42,585 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:42,585 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T18:49:42,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-01T18:49:42,588 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:42,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:42,589 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T18:49:42,592 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T18:49:42,593 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T18:49:42,598 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T18:49:42,598 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T18:49:42,601 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T18:49:42,604 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T18:49:42,609 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:49:42,610 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59818731, jitterRate=-0.1086314469575882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T18:49:42,616 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733078982519Initializing all the Stores at 1733078982521 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733078982522 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078982522Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078982523 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078982523Cleaning up temporary data from old regions at 1733078982598 (+75 ms)Region opened successfully at 1733078982616 (+18 ms) 2024-12-01T18:49:42,617 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-01T18:49:42,655 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37d189a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ec37ae3355e/172.17.0.2:0 2024-12-01T18:49:42,690 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-01T18:49:42,701 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-01T18:49:42,701 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-01T18:49:42,704 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-01T18:49:42,706 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-01T18:49:42,711 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-01T18:49:42,711 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-01T18:49:42,737 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-01T18:49:42,745 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-01T18:49:42,747 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-01T18:49:42,749 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-01T18:49:42,751 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-01T18:49:42,752 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-01T18:49:42,754 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-01T18:49:42,758 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-01T18:49:42,760 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-01T18:49:42,761 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-01T18:49:42,762 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-01T18:49:42,779 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-01T18:49:42,781 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-01T18:49:42,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T18:49:42,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T18:49:42,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T18:49:42,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T18:49:42,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,788 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9ec37ae3355e,40221,1733078981110, sessionid=0x1016f5c81760000, setting cluster-up flag (Was=false) 2024-12-01T18:49:42,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,810 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-01T18:49:42,811 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9ec37ae3355e,40221,1733078981110 2024-12-01T18:49:42,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:42,823 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-01T18:49:42,824 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9ec37ae3355e,40221,1733078981110 2024-12-01T18:49:42,830 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-01T18:49:42,881 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(746): ClusterId : bd982dac-d7cc-4a21-a476-59f23a0409a3 2024-12-01T18:49:42,881 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(746): ClusterId : bd982dac-d7cc-4a21-a476-59f23a0409a3 2024-12-01T18:49:42,881 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(746): ClusterId : bd982dac-d7cc-4a21-a476-59f23a0409a3 2024-12-01T18:49:42,884 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T18:49:42,884 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T18:49:42,884 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T18:49:42,890 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T18:49:42,890 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T18:49:42,890 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T18:49:42,890 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T18:49:42,890 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T18:49:42,890 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T18:49:42,897 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T18:49:42,898 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T18:49:42,898 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T18:49:42,898 DEBUG [RS:1;9ec37ae3355e:35069 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60eca280, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ec37ae3355e/172.17.0.2:0 2024-12-01T18:49:42,898 DEBUG [RS:2;9ec37ae3355e:34539 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60562013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ec37ae3355e/172.17.0.2:0 2024-12-01T18:49:42,900 DEBUG [RS:0;9ec37ae3355e:37405 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f5c6943, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ec37ae3355e/172.17.0.2:0 2024-12-01T18:49:42,908 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-01T18:49:42,919 DEBUG [RS:0;9ec37ae3355e:37405 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9ec37ae3355e:37405 2024-12-01T18:49:42,919 DEBUG [RS:1;9ec37ae3355e:35069 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9ec37ae3355e:35069 2024-12-01T18:49:42,919 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-01T18:49:42,921 DEBUG [RS:2;9ec37ae3355e:34539 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;9ec37ae3355e:34539 2024-12-01T18:49:42,924 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T18:49:42,924 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T18:49:42,924 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T18:49:42,924 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T18:49:42,924 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T18:49:42,924 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T18:49:42,924 DEBUG [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T18:49:42,924 DEBUG [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T18:49:42,924 DEBUG [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T18:49:42,928 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ec37ae3355e,40221,1733078981110 with port=34539, startcode=1733078981952 2024-12-01T18:49:42,928 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ec37ae3355e,40221,1733078981110 with port=35069, startcode=1733078981915 2024-12-01T18:49:42,928 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ec37ae3355e,40221,1733078981110 with port=37405, startcode=1733078981806 2024-12-01T18:49:42,930 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-01T18:49:42,938 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9ec37ae3355e,40221,1733078981110 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-01T18:49:42,943 DEBUG [RS:2;9ec37ae3355e:34539 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T18:49:42,943 DEBUG [RS:0;9ec37ae3355e:37405 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T18:49:42,943 DEBUG [RS:1;9ec37ae3355e:35069 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T18:49:42,948 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9ec37ae3355e:0, corePoolSize=5, maxPoolSize=5 2024-12-01T18:49:42,948 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9ec37ae3355e:0, corePoolSize=5, maxPoolSize=5 2024-12-01T18:49:42,948 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9ec37ae3355e:0, corePoolSize=5, maxPoolSize=5 2024-12-01T18:49:42,949 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9ec37ae3355e:0, corePoolSize=5, maxPoolSize=5 2024-12-01T18:49:42,949 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9ec37ae3355e:0, corePoolSize=10, maxPoolSize=10 2024-12-01T18:49:42,949 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:42,949 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9ec37ae3355e:0, corePoolSize=2, maxPoolSize=2 2024-12-01T18:49:42,950 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:42,954 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733079012954 2024-12-01T18:49:42,956 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-01T18:49:42,957 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-01T18:49:42,961 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-01T18:49:42,961 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-01T18:49:42,962 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-01T18:49:42,962 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-01T18:49:42,963 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:42,966 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-01T18:49:42,967 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-01T18:49:42,968 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-01T18:49:42,970 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-01T18:49:42,971 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-01T18:49:42,976 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9ec37ae3355e:0:becomeActiveMaster-HFileCleaner.large.0-1733078982974,5,FailOnTimeoutGroup] 2024-12-01T18:49:42,979 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9ec37ae3355e:0:becomeActiveMaster-HFileCleaner.small.0-1733078982977,5,FailOnTimeoutGroup] 2024-12-01T18:49:42,980 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:42,980 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-01T18:49:42,980 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T18:49:42,981 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-01T18:49:42,981 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:42,982 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:42,985 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54813, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T18:49:42,985 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58421, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T18:49:42,985 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35061, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T18:49:42,988 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:42,988 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-01T18:49:42,993 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40221 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9ec37ae3355e,37405,1733078981806 2024-12-01T18:49:42,995 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40221 {}] master.ServerManager(517): Registering regionserver=9ec37ae3355e,37405,1733078981806 2024-12-01T18:49:43,033 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40221 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:43,034 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40221 {}] master.ServerManager(517): Registering regionserver=9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:43,039 DEBUG [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018 2024-12-01T18:49:43,039 DEBUG [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38241 2024-12-01T18:49:43,039 DEBUG [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T18:49:43,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741831_1007 (size=1321) 2024-12-01T18:49:43,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741831_1007 (size=1321) 2024-12-01T18:49:43,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741831_1007 (size=1321) 2024-12-01T18:49:43,041 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40221 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9ec37ae3355e,35069,1733078981915 2024-12-01T18:49:43,041 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40221 {}] master.ServerManager(517): Registering regionserver=9ec37ae3355e,35069,1733078981915 2024-12-01T18:49:43,041 DEBUG [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018 2024-12-01T18:49:43,041 DEBUG [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38241 2024-12-01T18:49:43,042 DEBUG [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T18:49:43,042 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-01T18:49:43,042 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018 2024-12-01T18:49:43,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T18:49:43,045 DEBUG [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018 2024-12-01T18:49:43,045 DEBUG [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38241 2024-12-01T18:49:43,045 DEBUG [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T18:49:43,053 DEBUG [RS:0;9ec37ae3355e:37405 {}] zookeeper.ZKUtil(111): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9ec37ae3355e,37405,1733078981806 2024-12-01T18:49:43,053 WARN [RS:0;9ec37ae3355e:37405 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T18:49:43,054 INFO [RS:0;9ec37ae3355e:37405 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:43,054 DEBUG [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,37405,1733078981806 2024-12-01T18:49:43,055 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9ec37ae3355e,37405,1733078981806] 2024-12-01T18:49:43,055 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9ec37ae3355e,34539,1733078981952] 2024-12-01T18:49:43,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T18:49:43,056 DEBUG [RS:2;9ec37ae3355e:34539 {}] zookeeper.ZKUtil(111): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:43,056 WARN [RS:2;9ec37ae3355e:34539 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T18:49:43,057 INFO [RS:2;9ec37ae3355e:34539 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:43,057 DEBUG [RS:1;9ec37ae3355e:35069 {}] zookeeper.ZKUtil(111): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9ec37ae3355e,35069,1733078981915 2024-12-01T18:49:43,057 WARN [RS:1;9ec37ae3355e:35069 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T18:49:43,057 DEBUG [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:43,057 INFO [RS:1;9ec37ae3355e:35069 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:43,057 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9ec37ae3355e,35069,1733078981915] 2024-12-01T18:49:43,057 DEBUG [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,35069,1733078981915 2024-12-01T18:49:43,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741832_1008 (size=32) 2024-12-01T18:49:43,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741832_1008 (size=32) 2024-12-01T18:49:43,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741832_1008 (size=32) 2024-12-01T18:49:43,082 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:43,085 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-01T18:49:43,086 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T18:49:43,086 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T18:49:43,086 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T18:49:43,087 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T18:49:43,088 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:43,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:49:43,089 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-01T18:49:43,091 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T18:49:43,091 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:43,092 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:49:43,092 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-01T18:49:43,095 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T18:49:43,095 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:43,096 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:49:43,096 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-01T18:49:43,098 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T18:49:43,099 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:43,099 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:49:43,100 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T18:49:43,101 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740 2024-12-01T18:49:43,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740 2024-12-01T18:49:43,106 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T18:49:43,106 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T18:49:43,106 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T18:49:43,106 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T18:49:43,106 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T18:49:43,107 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T18:49:43,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T18:49:43,114 INFO [RS:1;9ec37ae3355e:35069 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T18:49:43,114 INFO [RS:2;9ec37ae3355e:34539 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T18:49:43,114 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,114 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,117 INFO [RS:0;9ec37ae3355e:37405 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T18:49:43,117 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,117 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T18:49:43,117 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T18:49:43,120 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T18:49:43,121 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:49:43,122 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70807215, jitterRate=0.05510972440242767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T18:49:43,124 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733078983082Initializing all the Stores at 1733078983084 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733078983084Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733078983085 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078983085Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733078983085Cleaning up temporary data from old regions at 1733078983106 (+21 ms)Region opened successfully at 1733078983124 (+18 ms) 2024-12-01T18:49:43,124 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T18:49:43,124 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T18:49:43,124 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T18:49:43,124 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T18:49:43,124 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T18:49:43,124 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T18:49:43,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T18:49:43,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T18:49:43,126 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,126 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,126 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T18:49:43,126 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733078983124Disabling compacts and flushes for region at 1733078983124Disabling writes for close at 1733078983125 (+1 ms)Writing region close event to WAL at 1733078983126 (+1 ms)Closed at 1733078983126 2024-12-01T18:49:43,126 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,126 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,126 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,127 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0, corePoolSize=2, maxPoolSize=2 2024-12-01T18:49:43,127 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0, corePoolSize=2, maxPoolSize=2 2024-12-01T18:49:43,128 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,128 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,128 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,128 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,128 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,128 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,128 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,128 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,128 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,128 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,128 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9ec37ae3355e:0, corePoolSize=3, maxPoolSize=3 2024-12-01T18:49:43,128 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,128 DEBUG [RS:1;9ec37ae3355e:35069 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9ec37ae3355e:0, corePoolSize=3, maxPoolSize=3 2024-12-01T18:49:43,129 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,129 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9ec37ae3355e:0, corePoolSize=3, maxPoolSize=3 2024-12-01T18:49:43,129 DEBUG [RS:2;9ec37ae3355e:34539 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9ec37ae3355e:0, corePoolSize=3, maxPoolSize=3 2024-12-01T18:49:43,129 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0, corePoolSize=2, maxPoolSize=2 2024-12-01T18:49:43,130 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,130 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,130 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,130 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T18:49:43,130 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-01T18:49:43,130 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,130 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,130 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9ec37ae3355e:0, corePoolSize=1, maxPoolSize=1 2024-12-01T18:49:43,131 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9ec37ae3355e:0, corePoolSize=3, maxPoolSize=3 2024-12-01T18:49:43,131 DEBUG [RS:0;9ec37ae3355e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9ec37ae3355e:0, corePoolSize=3, maxPoolSize=3 2024-12-01T18:49:43,131 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,131 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,131 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,132 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,132 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,132 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,132 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,132 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,132 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,132 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,132 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ec37ae3355e,35069,1733078981915-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T18:49:43,132 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ec37ae3355e,34539,1733078981952-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T18:49:43,134 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,134 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,134 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,134 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,134 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,136 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ec37ae3355e,37405,1733078981806-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T18:49:43,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T18:49:43,146 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T18:49:43,151 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-01T18:49:43,157 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T18:49:43,158 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T18:49:43,159 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T18:49:43,159 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ec37ae3355e,37405,1733078981806-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,159 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ec37ae3355e,35069,1733078981915-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,159 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ec37ae3355e,34539,1733078981952-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,160 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,160 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,160 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,160 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.Replication(171): 9ec37ae3355e,37405,1733078981806 started 2024-12-01T18:49:43,160 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.Replication(171): 9ec37ae3355e,34539,1733078981952 started 2024-12-01T18:49:43,160 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.Replication(171): 9ec37ae3355e,35069,1733078981915 started 2024-12-01T18:49:43,177 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,178 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(1482): Serving as 9ec37ae3355e,35069,1733078981915, RpcServer on 9ec37ae3355e/172.17.0.2:35069, sessionid=0x1016f5c81760002 2024-12-01T18:49:43,178 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,178 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(1482): Serving as 9ec37ae3355e,34539,1733078981952, RpcServer on 9ec37ae3355e/172.17.0.2:34539, sessionid=0x1016f5c81760003 2024-12-01T18:49:43,178 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T18:49:43,178 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T18:49:43,179 DEBUG [RS:1;9ec37ae3355e:35069 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9ec37ae3355e,35069,1733078981915 2024-12-01T18:49:43,179 DEBUG [RS:2;9ec37ae3355e:34539 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:43,179 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ec37ae3355e,35069,1733078981915' 2024-12-01T18:49:43,179 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ec37ae3355e,34539,1733078981952' 2024-12-01T18:49:43,179 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T18:49:43,179 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T18:49:43,180 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T18:49:43,180 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T18:49:43,181 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T18:49:43,181 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T18:49:43,181 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T18:49:43,181 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T18:49:43,181 DEBUG [RS:1;9ec37ae3355e:35069 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9ec37ae3355e,35069,1733078981915 2024-12-01T18:49:43,181 DEBUG [RS:2;9ec37ae3355e:34539 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:43,181 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ec37ae3355e,35069,1733078981915' 2024-12-01T18:49:43,181 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ec37ae3355e,34539,1733078981952' 2024-12-01T18:49:43,181 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T18:49:43,181 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T18:49:43,182 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T18:49:43,182 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T18:49:43,182 DEBUG [RS:1;9ec37ae3355e:35069 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T18:49:43,182 DEBUG [RS:2;9ec37ae3355e:34539 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T18:49:43,182 INFO [RS:1;9ec37ae3355e:35069 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T18:49:43,182 INFO [RS:2;9ec37ae3355e:34539 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T18:49:43,182 INFO [RS:1;9ec37ae3355e:35069 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T18:49:43,183 INFO [RS:2;9ec37ae3355e:34539 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T18:49:43,184 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:43,185 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(1482): Serving as 9ec37ae3355e,37405,1733078981806, RpcServer on 9ec37ae3355e/172.17.0.2:37405, sessionid=0x1016f5c81760001 2024-12-01T18:49:43,185 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T18:49:43,185 DEBUG [RS:0;9ec37ae3355e:37405 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9ec37ae3355e,37405,1733078981806 2024-12-01T18:49:43,185 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ec37ae3355e,37405,1733078981806' 2024-12-01T18:49:43,185 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T18:49:43,186 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T18:49:43,186 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T18:49:43,186 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T18:49:43,186 DEBUG [RS:0;9ec37ae3355e:37405 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9ec37ae3355e,37405,1733078981806 2024-12-01T18:49:43,186 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ec37ae3355e,37405,1733078981806' 2024-12-01T18:49:43,186 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T18:49:43,187 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T18:49:43,188 DEBUG [RS:0;9ec37ae3355e:37405 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T18:49:43,188 INFO [RS:0;9ec37ae3355e:37405 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T18:49:43,188 INFO [RS:0;9ec37ae3355e:37405 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T18:49:43,288 INFO [RS:1;9ec37ae3355e:35069 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:49:43,288 INFO [RS:2;9ec37ae3355e:34539 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:49:43,289 INFO [RS:0;9ec37ae3355e:37405 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:49:43,291 INFO [RS:1;9ec37ae3355e:35069 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ec37ae3355e%2C35069%2C1733078981915, suffix=, logDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,35069,1733078981915, archiveDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/oldWALs, maxLogs=32 2024-12-01T18:49:43,291 INFO [RS:0;9ec37ae3355e:37405 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ec37ae3355e%2C37405%2C1733078981806, suffix=, logDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,37405,1733078981806, archiveDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/oldWALs, maxLogs=32 2024-12-01T18:49:43,291 INFO [RS:2;9ec37ae3355e:34539 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ec37ae3355e%2C34539%2C1733078981952, suffix=, logDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952, archiveDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/oldWALs, maxLogs=32 2024-12-01T18:49:43,301 WARN [9ec37ae3355e:40221 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T18:49:43,310 DEBUG [RS:2;9ec37ae3355e:34539 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952/9ec37ae3355e%2C34539%2C1733078981952.1733078983296, exclude list is [], retry=0 2024-12-01T18:49:43,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:43,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:43,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:43,317 DEBUG [RS:0;9ec37ae3355e:37405 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,37405,1733078981806/9ec37ae3355e%2C37405%2C1733078981806.1733078983296, exclude list is [], retry=0 2024-12-01T18:49:43,317 DEBUG [RS:1;9ec37ae3355e:35069 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,35069,1733078981915/9ec37ae3355e%2C35069%2C1733078981915.1733078983296, exclude list is [], retry=0 2024-12-01T18:49:43,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:43,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:43,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:43,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:43,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:43,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:43,356 INFO [RS:0;9ec37ae3355e:37405 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,37405,1733078981806/9ec37ae3355e%2C37405%2C1733078981806.1733078983296 2024-12-01T18:49:43,356 INFO [RS:2;9ec37ae3355e:34539 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952/9ec37ae3355e%2C34539%2C1733078981952.1733078983296 2024-12-01T18:49:43,357 INFO [RS:1;9ec37ae3355e:35069 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,35069,1733078981915/9ec37ae3355e%2C35069%2C1733078981915.1733078983296 2024-12-01T18:49:43,357 DEBUG [RS:0;9ec37ae3355e:37405 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:49:43,358 DEBUG [RS:2;9ec37ae3355e:34539 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:40921:40921)] 2024-12-01T18:49:43,358 DEBUG [RS:1;9ec37ae3355e:35069 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:33601:33601)] 2024-12-01T18:49:43,554 DEBUG [9ec37ae3355e:40221 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-01T18:49:43,562 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(204): Hosts are {9ec37ae3355e=0} racks are {/default-rack=0} 2024-12-01T18:49:43,569 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T18:49:43,569 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T18:49:43,569 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-01T18:49:43,569 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T18:49:43,569 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T18:49:43,570 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-01T18:49:43,570 INFO [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T18:49:43,570 INFO [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T18:49:43,570 INFO [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-01T18:49:43,570 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T18:49:43,577 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:43,583 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ec37ae3355e,34539,1733078981952, state=OPENING 2024-12-01T18:49:43,588 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-01T18:49:43,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:43,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:43,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:43,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:49:43,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:49:43,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:49:43,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:49:43,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:49:43,593 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T18:49:43,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9ec37ae3355e,34539,1733078981952}] 2024-12-01T18:49:43,771 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T18:49:43,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46737, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T18:49:43,786 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T18:49:43,787 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:43,787 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T18:49:43,791 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ec37ae3355e%2C34539%2C1733078981952.meta, suffix=.meta, logDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952, archiveDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/oldWALs, maxLogs=32 2024-12-01T18:49:43,807 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta, exclude list is [], retry=0 2024-12-01T18:49:43,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:43,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:43,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:43,843 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta 2024-12-01T18:49:43,844 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:49:43,844 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:43,846 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T18:49:43,849 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T18:49:43,855 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T18:49:43,861 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T18:49:43,862 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:43,862 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T18:49:43,862 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T18:49:43,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-01T18:49:43,867 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T18:49:43,867 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:43,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:49:43,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-01T18:49:43,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T18:49:43,869 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:43,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:49:43,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-01T18:49:43,871 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T18:49:43,871 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:43,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:49:43,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-01T18:49:43,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T18:49:43,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:43,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:49:43,875 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T18:49:43,876 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740 2024-12-01T18:49:43,879 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740 2024-12-01T18:49:43,882 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T18:49:43,882 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T18:49:43,883 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T18:49:43,886 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T18:49:43,888 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66234454, jitterRate=-0.013029724359512329}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T18:49:43,888 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T18:49:43,890 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733078983862Writing region info on filesystem at 1733078983863 (+1 ms)Initializing all the Stores at 1733078983865 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733078983865Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733078983865Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078983865Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733078983865Cleaning up temporary data from old regions at 1733078983882 (+17 ms)Running coprocessor post-open hooks at 1733078983888 (+6 ms)Region opened successfully at 1733078983890 (+2 ms) 2024-12-01T18:49:43,900 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733078983761 2024-12-01T18:49:43,912 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T18:49:43,913 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T18:49:43,915 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:43,917 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ec37ae3355e,34539,1733078981952, state=OPEN 2024-12-01T18:49:43,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T18:49:43,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T18:49:43,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T18:49:43,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T18:49:43,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:49:43,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:49:43,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:49:43,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:49:43,920 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:43,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-01T18:49:43,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9ec37ae3355e,34539,1733078981952 in 326 msec 2024-12-01T18:49:43,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-01T18:49:43,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 790 msec 2024-12-01T18:49:43,934 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T18:49:43,934 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-01T18:49:43,956 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T18:49:43,958 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1] 2024-12-01T18:49:43,982 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T18:49:43,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34489, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T18:49:44,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1440 sec 2024-12-01T18:49:44,005 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733078984005, completionTime=-1 2024-12-01T18:49:44,008 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-01T18:49:44,008 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-01T18:49:44,035 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-01T18:49:44,035 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733079044035 2024-12-01T18:49:44,035 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733079104035 2024-12-01T18:49:44,035 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-01T18:49:44,037 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-01T18:49:44,043 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ec37ae3355e,40221,1733078981110-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:44,044 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ec37ae3355e,40221,1733078981110-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:44,044 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ec37ae3355e,40221,1733078981110-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:44,045 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9ec37ae3355e:40221, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:44,046 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:44,046 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:44,052 DEBUG [master/9ec37ae3355e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-01T18:49:44,095 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.061sec 2024-12-01T18:49:44,097 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-01T18:49:44,098 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-01T18:49:44,099 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-01T18:49:44,099 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-01T18:49:44,100 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-01T18:49:44,100 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ec37ae3355e,40221,1733078981110-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T18:49:44,101 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ec37ae3355e,40221,1733078981110-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-01T18:49:44,105 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-01T18:49:44,106 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-01T18:49:44,106 INFO [master/9ec37ae3355e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ec37ae3355e,40221,1733078981110-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:44,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@287698dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T18:49:44,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ec37ae3355e,40221,-1 for getting cluster id 2024-12-01T18:49:44,218 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T18:49:44,228 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bd982dac-d7cc-4a21-a476-59f23a0409a3' 2024-12-01T18:49:44,231 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T18:49:44,231 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bd982dac-d7cc-4a21-a476-59f23a0409a3" 2024-12-01T18:49:44,231 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42346407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T18:49:44,232 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ec37ae3355e,40221,-1] 2024-12-01T18:49:44,235 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T18:49:44,237 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T18:49:44,240 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T18:49:44,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd88229, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T18:49:44,244 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T18:49:44,252 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1] 2024-12-01T18:49:44,253 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T18:49:44,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T18:49:44,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9ec37ae3355e,40221,1733078981110 2024-12-01T18:49:44,278 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:38241/hbase 2024-12-01T18:49:44,293 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=365, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=197, ProcessCount=11, AvailableMemoryMB=4205 2024-12-01T18:49:44,329 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:44,333 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:44,334 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:49:44,339 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-65359202, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/hregion-65359202, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:44,367 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-65359202/hregion-65359202.1733078984342, exclude list is [], retry=0 2024-12-01T18:49:44,372 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:44,372 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:44,372 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:44,409 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-65359202/hregion-65359202.1733078984342 2024-12-01T18:49:44,409 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921)] 2024-12-01T18:49:44,409 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 06d2bdb5c1176c7202771dd6b01c1d1f, NAME => 'testReplayEditsWrittenIntoWAL,,1733078984330.06d2bdb5c1176c7202771dd6b01c1d1f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38241/hbase 2024-12-01T18:49:44,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741838_1014 (size=64) 2024-12-01T18:49:44,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741838_1014 (size=64) 2024-12-01T18:49:44,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741838_1014 (size=64) 2024-12-01T18:49:44,429 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733078984330.06d2bdb5c1176c7202771dd6b01c1d1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:44,431 INFO [StoreOpener-06d2bdb5c1176c7202771dd6b01c1d1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,433 INFO [StoreOpener-06d2bdb5c1176c7202771dd6b01c1d1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06d2bdb5c1176c7202771dd6b01c1d1f columnFamilyName a 2024-12-01T18:49:44,434 DEBUG [StoreOpener-06d2bdb5c1176c7202771dd6b01c1d1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:44,435 INFO [StoreOpener-06d2bdb5c1176c7202771dd6b01c1d1f-1 {}] regionserver.HStore(327): Store=06d2bdb5c1176c7202771dd6b01c1d1f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:44,435 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,436 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,437 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,437 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,437 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,440 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,443 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:49:44,444 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 06d2bdb5c1176c7202771dd6b01c1d1f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70427284, jitterRate=0.04944831132888794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T18:49:44,445 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 06d2bdb5c1176c7202771dd6b01c1d1f: Writing region info on filesystem at 1733078984429Initializing all the Stores at 1733078984430 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078984430Cleaning up temporary data from old regions at 1733078984437 (+7 ms)Region opened successfully at 1733078984445 (+8 ms) 2024-12-01T18:49:44,445 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 06d2bdb5c1176c7202771dd6b01c1d1f, disabling compactions & flushes 2024-12-01T18:49:44,445 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733078984330.06d2bdb5c1176c7202771dd6b01c1d1f. 2024-12-01T18:49:44,445 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733078984330.06d2bdb5c1176c7202771dd6b01c1d1f. 2024-12-01T18:49:44,445 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733078984330.06d2bdb5c1176c7202771dd6b01c1d1f. after waiting 0 ms 2024-12-01T18:49:44,445 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733078984330.06d2bdb5c1176c7202771dd6b01c1d1f. 2024-12-01T18:49:44,446 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733078984330.06d2bdb5c1176c7202771dd6b01c1d1f. 2024-12-01T18:49:44,446 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 06d2bdb5c1176c7202771dd6b01c1d1f: Waiting for close lock at 1733078984445Disabling compacts and flushes for region at 1733078984445Disabling writes for close at 1733078984445Writing region close event to WAL at 1733078984446 (+1 ms)Closed at 1733078984446 2024-12-01T18:49:44,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741837_1013 (size=175) 2024-12-01T18:49:44,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741837_1013 (size=175) 2024-12-01T18:49:44,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741837_1013 (size=175) 2024-12-01T18:49:44,457 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:49:44,457 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-65359202:(num 1733078984342) 2024-12-01T18:49:44,459 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-01T18:49:44,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741839_1015 (size=469) 2024-12-01T18:49:44,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741839_1015 (size=469) 2024-12-01T18:49:44,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741839_1015 (size=469) 2024-12-01T18:49:44,504 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-01T18:49:44,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741840_1016 (size=322) 2024-12-01T18:49:44,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741840_1016 (size=322) 2024-12-01T18:49:44,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741840_1016 (size=322) 2024-12-01T18:49:44,566 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-1, size=469 (469bytes) 2024-12-01T18:49:44,567 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-01T18:49:44,567 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-01T18:49:44,568 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-1 2024-12-01T18:49:44,573 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-1 after 3ms 2024-12-01T18:49:44,604 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:44,604 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-1 took 39ms 2024-12-01T18:49:44,609 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-1 so closing down 2024-12-01T18:49:44,610 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:49:44,612 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-01T18:49:44,641 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000001-wal-1.temp 2024-12-01T18:49:44,642 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:49:44,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741841_1017 (size=469) 2024-12-01T18:49:44,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741841_1017 (size=469) 2024-12-01T18:49:44,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741841_1017 (size=469) 2024-12-01T18:49:44,658 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-01T18:49:44,661 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000002 2024-12-01T18:49:44,665 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 56 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-01T18:49:44,665 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-1, journal: Splitting hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-1, size=469 (469bytes) at 1733078984567Finishing writing output for hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-1 so closing down at 1733078984609 (+42 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000001-wal-1.temp at 1733078984641 (+32 ms)3 split writer threads finished at 1733078984642 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733078984659 (+17 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000002 at 1733078984661 (+2 ms)Processed 2 edits across 1 Regions in 56 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733078984665 (+4 ms) 2024-12-01T18:49:44,680 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-2, size=322 (322bytes) 2024-12-01T18:49:44,680 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-2 2024-12-01T18:49:44,681 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-2 after 1ms 2024-12-01T18:49:44,710 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:44,710 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-2 took 30ms 2024-12-01T18:49:44,713 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-2 so closing down 2024-12-01T18:49:44,713 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:49:44,716 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-01T18:49:44,743 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000002-wal-2.temp 2024-12-01T18:49:44,743 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:49:44,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741842_1018 (size=322) 2024-12-01T18:49:44,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741842_1018 (size=322) 2024-12-01T18:49:44,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741842_1018 (size=322) 2024-12-01T18:49:44,753 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-01T18:49:44,783 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:44,785 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-01T18:49:44,786 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 76 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-01T18:49:44,787 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-2, journal: Splitting hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-2, size=322 (322bytes) at 1733078984680Finishing writing output for hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-2 so closing down at 1733078984713 (+33 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000002-wal-2.temp at 1733078984743 (+30 ms)3 split writer threads finished at 1733078984743Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733078984753 (+10 ms)Processed 1 edits across 1 Regions in 76 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733078984786 (+33 ms) 2024-12-01T18:49:44,787 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:49:44,789 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:44,802 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal.1733078984790, exclude list is [], retry=0 2024-12-01T18:49:44,805 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:44,806 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:44,807 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:44,833 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal.1733078984790 2024-12-01T18:49:44,833 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:49:44,833 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 06d2bdb5c1176c7202771dd6b01c1d1f, NAME => 'testReplayEditsWrittenIntoWAL,,1733078984330.06d2bdb5c1176c7202771dd6b01c1d1f.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:44,834 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733078984330.06d2bdb5c1176c7202771dd6b01c1d1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:44,834 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,834 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,836 INFO [StoreOpener-06d2bdb5c1176c7202771dd6b01c1d1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,837 INFO [StoreOpener-06d2bdb5c1176c7202771dd6b01c1d1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06d2bdb5c1176c7202771dd6b01c1d1f columnFamilyName a 2024-12-01T18:49:44,838 DEBUG [StoreOpener-06d2bdb5c1176c7202771dd6b01c1d1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:44,839 INFO [StoreOpener-06d2bdb5c1176c7202771dd6b01c1d1f-1 {}] regionserver.HStore(327): Store=06d2bdb5c1176c7202771dd6b01c1d1f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:44,839 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,840 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,843 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:44,844 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000002 2024-12-01T18:49:44,872 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:44,879 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000002 2024-12-01T18:49:44,882 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 06d2bdb5c1176c7202771dd6b01c1d1f 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-01T18:49:44,932 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/.tmp/a/f0801ee3e65746a289902688a6d3f660 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733078984457/Put/seqid=0 2024-12-01T18:49:44,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741844_1020 (size=5170) 2024-12-01T18:49:44,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741844_1020 (size=5170) 2024-12-01T18:49:44,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741844_1020 (size=5170) 2024-12-01T18:49:44,948 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/.tmp/a/f0801ee3e65746a289902688a6d3f660 2024-12-01T18:49:44,990 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/.tmp/a/f0801ee3e65746a289902688a6d3f660 as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/a/f0801ee3e65746a289902688a6d3f660 2024-12-01T18:49:44,998 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/a/f0801ee3e65746a289902688a6d3f660, entries=2, sequenceid=2, filesize=5.0 K 2024-12-01T18:49:45,005 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 06d2bdb5c1176c7202771dd6b01c1d1f in 121ms, sequenceid=2, compaction requested=false; wal=null 2024-12-01T18:49:45,007 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/0000000000000000002 2024-12-01T18:49:45,007 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:45,007 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:45,011 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 06d2bdb5c1176c7202771dd6b01c1d1f 2024-12-01T18:49:45,014 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/06d2bdb5c1176c7202771dd6b01c1d1f/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-01T18:49:45,015 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 06d2bdb5c1176c7202771dd6b01c1d1f; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73821971, jitterRate=0.10003308951854706}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T18:49:45,017 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 06d2bdb5c1176c7202771dd6b01c1d1f: Writing region info on filesystem at 1733078984834Initializing all the Stores at 1733078984835 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078984835Obtaining lock to block concurrent updates at 1733078984882 (+47 ms)Preparing flush snapshotting stores in 06d2bdb5c1176c7202771dd6b01c1d1f at 1733078984882Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733078984330.06d2bdb5c1176c7202771dd6b01c1d1f., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733078984885 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733078984330.06d2bdb5c1176c7202771dd6b01c1d1f. at 1733078984885Flushing 06d2bdb5c1176c7202771dd6b01c1d1f/a: creating writer at 1733078984887 (+2 ms)Flushing 06d2bdb5c1176c7202771dd6b01c1d1f/a: appending metadata at 1733078984922 (+35 ms)Flushing 06d2bdb5c1176c7202771dd6b01c1d1f/a: closing flushed file at 1733078984924 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36ce6335: reopening flushed file at 1733078984988 (+64 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 06d2bdb5c1176c7202771dd6b01c1d1f in 121ms, sequenceid=2, compaction requested=false; wal=null at 1733078985005 (+17 ms)Cleaning up temporary data from old regions at 1733078985007 (+2 ms)Region opened successfully at 1733078985016 (+9 ms) 2024-12-01T18:49:45,041 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=376 (was 365) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:52218 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:58896 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:52336 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1540968591-172.17.0.2-1733078978096:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:38241/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1540968591-172.17.0.2-1733078978096:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1540968591-172.17.0.2-1733078978096:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:36642 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:58760 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=197 (was 197), ProcessCount=11 (was 11), AvailableMemoryMB=4190 (was 4205) 2024-12-01T18:49:45,052 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=197, ProcessCount=11, AvailableMemoryMB=4189 2024-12-01T18:49:45,068 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:45,070 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:45,071 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:49:45,075 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98495099, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/hregion-98495099, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:45,088 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98495099/hregion-98495099.1733078985076, exclude list is [], retry=0 2024-12-01T18:49:45,092 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:45,092 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:45,092 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:45,121 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98495099/hregion-98495099.1733078985076 2024-12-01T18:49:45,122 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601)] 2024-12-01T18:49:45,122 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7d0facacaee95a08b4560776cf7ad274, NAME => 'testReplayEditsWrittenIntoWAL,,1733078985068.7d0facacaee95a08b4560776cf7ad274.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38241/hbase 2024-12-01T18:49:45,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741846_1022 (size=64) 2024-12-01T18:49:45,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741846_1022 (size=64) 2024-12-01T18:49:45,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741846_1022 (size=64) 2024-12-01T18:49:45,134 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733078985068.7d0facacaee95a08b4560776cf7ad274.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:45,136 INFO [StoreOpener-7d0facacaee95a08b4560776cf7ad274-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,138 INFO [StoreOpener-7d0facacaee95a08b4560776cf7ad274-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d0facacaee95a08b4560776cf7ad274 columnFamilyName a 2024-12-01T18:49:45,139 DEBUG [StoreOpener-7d0facacaee95a08b4560776cf7ad274-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:45,139 INFO [StoreOpener-7d0facacaee95a08b4560776cf7ad274-1 {}] regionserver.HStore(327): Store=7d0facacaee95a08b4560776cf7ad274/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:45,139 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,140 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,141 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,141 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,141 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,143 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,146 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:49:45,147 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7d0facacaee95a08b4560776cf7ad274; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66855546, jitterRate=-0.0037747323513031006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T18:49:45,147 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7d0facacaee95a08b4560776cf7ad274: Writing region info on filesystem at 1733078985134Initializing all the Stores at 1733078985136 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078985136Cleaning up temporary data from old regions at 1733078985141 (+5 ms)Region opened successfully at 1733078985147 (+6 ms) 2024-12-01T18:49:45,147 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7d0facacaee95a08b4560776cf7ad274, disabling compactions & flushes 2024-12-01T18:49:45,147 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733078985068.7d0facacaee95a08b4560776cf7ad274. 2024-12-01T18:49:45,147 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733078985068.7d0facacaee95a08b4560776cf7ad274. 2024-12-01T18:49:45,147 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733078985068.7d0facacaee95a08b4560776cf7ad274. after waiting 0 ms 2024-12-01T18:49:45,147 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733078985068.7d0facacaee95a08b4560776cf7ad274. 2024-12-01T18:49:45,148 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733078985068.7d0facacaee95a08b4560776cf7ad274. 2024-12-01T18:49:45,148 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7d0facacaee95a08b4560776cf7ad274: Waiting for close lock at 1733078985147Disabling compacts and flushes for region at 1733078985147Disabling writes for close at 1733078985147Writing region close event to WAL at 1733078985148 (+1 ms)Closed at 1733078985148 2024-12-01T18:49:45,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741845_1021 (size=175) 2024-12-01T18:49:45,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741845_1021 (size=175) 2024-12-01T18:49:45,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741845_1021 (size=175) 2024-12-01T18:49:45,157 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:49:45,157 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98495099:(num 1733078985076) 2024-12-01T18:49:45,158 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-01T18:49:45,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741847_1023 (size=469) 2024-12-01T18:49:45,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741847_1023 (size=469) 2024-12-01T18:49:45,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741847_1023 (size=469) 2024-12-01T18:49:45,198 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-01T18:49:45,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741848_1024 (size=322) 2024-12-01T18:49:45,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741848_1024 (size=322) 2024-12-01T18:49:45,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741848_1024 (size=322) 2024-12-01T18:49:45,250 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-2, size=322 (322bytes) 2024-12-01T18:49:45,250 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-2 2024-12-01T18:49:45,251 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-2 after 1ms 2024-12-01T18:49:45,281 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:45,281 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-2 took 31ms 2024-12-01T18:49:45,283 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-2 so closing down 2024-12-01T18:49:45,283 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:49:45,286 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-01T18:49:45,311 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002-wal-2.temp 2024-12-01T18:49:45,312 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:49:45,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741849_1025 (size=322) 2024-12-01T18:49:45,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741849_1025 (size=322) 2024-12-01T18:49:45,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741849_1025 (size=322) 2024-12-01T18:49:45,321 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-01T18:49:45,323 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002 2024-12-01T18:49:45,323 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-01T18:49:45,323 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-2, journal: Splitting hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-2, size=322 (322bytes) at 1733078985250Finishing writing output for hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-2 so closing down at 1733078985283 (+33 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002-wal-2.temp at 1733078985311 (+28 ms)3 split writer threads finished at 1733078985312 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733078985321 (+9 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002 at 1733078985323 (+2 ms)Processed 1 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733078985323 2024-12-01T18:49:45,337 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-1, size=469 (469bytes) 2024-12-01T18:49:45,337 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-1 2024-12-01T18:49:45,338 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-1 after 1ms 2024-12-01T18:49:45,366 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:45,366 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-1 took 29ms 2024-12-01T18:49:45,369 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-1 so closing down 2024-12-01T18:49:45,369 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:49:45,371 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-01T18:49:45,399 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000001-wal-1.temp 2024-12-01T18:49:45,400 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:49:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741850_1026 (size=469) 2024-12-01T18:49:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741850_1026 (size=469) 2024-12-01T18:49:45,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741850_1026 (size=469) 2024-12-01T18:49:45,408 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-01T18:49:45,438 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:45,440 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002, length=322 2024-12-01T18:49:45,443 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002 2024-12-01T18:49:45,443 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-01T18:49:45,443 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-1, journal: Splitting hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-1, size=469 (469bytes) at 1733078985337Finishing writing output for hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-1 so closing down at 1733078985369 (+32 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000001-wal-1.temp at 1733078985399 (+30 ms)3 split writer threads finished at 1733078985400 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733078985408 (+8 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002 at 1733078985443 (+35 ms)Processed 2 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733078985443 2024-12-01T18:49:45,443 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:49:45,445 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:45,461 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal.1733078985446, exclude list is [], retry=0 2024-12-01T18:49:45,464 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:45,465 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:45,465 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:45,493 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal.1733078985446 2024-12-01T18:49:45,493 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:49:45,493 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7d0facacaee95a08b4560776cf7ad274, NAME => 'testReplayEditsWrittenIntoWAL,,1733078985068.7d0facacaee95a08b4560776cf7ad274.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:45,494 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733078985068.7d0facacaee95a08b4560776cf7ad274.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:45,494 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,494 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,496 INFO [StoreOpener-7d0facacaee95a08b4560776cf7ad274-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,498 INFO [StoreOpener-7d0facacaee95a08b4560776cf7ad274-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d0facacaee95a08b4560776cf7ad274 columnFamilyName a 2024-12-01T18:49:45,498 DEBUG [StoreOpener-7d0facacaee95a08b4560776cf7ad274-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:45,499 INFO [StoreOpener-7d0facacaee95a08b4560776cf7ad274-1 {}] regionserver.HStore(327): Store=7d0facacaee95a08b4560776cf7ad274/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:45,499 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,500 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,502 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,503 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002 2024-12-01T18:49:45,532 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:45,534 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002 2024-12-01T18:49:45,534 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7d0facacaee95a08b4560776cf7ad274 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-01T18:49:45,551 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/.tmp/a/1494beb01db34cdaa3e607a2abf841ce is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733078985158/Put/seqid=0 2024-12-01T18:49:45,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741852_1028 (size=5170) 2024-12-01T18:49:45,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741852_1028 (size=5170) 2024-12-01T18:49:45,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741852_1028 (size=5170) 2024-12-01T18:49:45,561 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/.tmp/a/1494beb01db34cdaa3e607a2abf841ce 2024-12-01T18:49:45,571 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/.tmp/a/1494beb01db34cdaa3e607a2abf841ce as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/a/1494beb01db34cdaa3e607a2abf841ce 2024-12-01T18:49:45,578 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/a/1494beb01db34cdaa3e607a2abf841ce, entries=2, sequenceid=2, filesize=5.0 K 2024-12-01T18:49:45,578 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 7d0facacaee95a08b4560776cf7ad274 in 44ms, sequenceid=2, compaction requested=false; wal=null 2024-12-01T18:49:45,579 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/0000000000000000002 2024-12-01T18:49:45,580 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,580 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,582 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7d0facacaee95a08b4560776cf7ad274 2024-12-01T18:49:45,584 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/7d0facacaee95a08b4560776cf7ad274/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-01T18:49:45,585 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7d0facacaee95a08b4560776cf7ad274; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66094014, jitterRate=-0.015122443437576294}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T18:49:45,586 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7d0facacaee95a08b4560776cf7ad274: Writing region info on filesystem at 1733078985494Initializing all the Stores at 1733078985495 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078985495Obtaining lock to block concurrent updates at 1733078985534 (+39 ms)Preparing flush snapshotting stores in 7d0facacaee95a08b4560776cf7ad274 at 1733078985534Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733078985068.7d0facacaee95a08b4560776cf7ad274., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733078985534Flushing stores of testReplayEditsWrittenIntoWAL,,1733078985068.7d0facacaee95a08b4560776cf7ad274. at 1733078985534Flushing 7d0facacaee95a08b4560776cf7ad274/a: creating writer at 1733078985534Flushing 7d0facacaee95a08b4560776cf7ad274/a: appending metadata at 1733078985550 (+16 ms)Flushing 7d0facacaee95a08b4560776cf7ad274/a: closing flushed file at 1733078985550Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37e5122b: reopening flushed file at 1733078985570 (+20 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 7d0facacaee95a08b4560776cf7ad274 in 44ms, sequenceid=2, compaction requested=false; wal=null at 1733078985578 (+8 ms)Cleaning up temporary data from old regions at 1733078985580 (+2 ms)Region opened successfully at 1733078985586 (+6 ms) 2024-12-01T18:49:45,602 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: PacketResponder: BP-1540968591-172.17.0.2-1733078978096:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:58962 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:36718 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:52218 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1540968591-172.17.0.2-1733078978096:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:36712 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1540968591-172.17.0.2-1733078978096:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:58760 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:52412 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=197 (was 197), ProcessCount=11 (was 11), AvailableMemoryMB=4175 (was 4189) 2024-12-01T18:49:45,613 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=197, ProcessCount=11, AvailableMemoryMB=4174 2024-12-01T18:49:45,628 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:45,630 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:45,631 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:49:45,634 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-95754250, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/hregion-95754250, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:45,647 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-95754250/hregion-95754250.1733078985635, exclude list is [], retry=0 2024-12-01T18:49:45,651 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:45,651 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:45,652 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:45,680 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-95754250/hregion-95754250.1733078985635 2024-12-01T18:49:45,680 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:49:45,681 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => fde08b92802f1a5199c8495f09f916a0, NAME => 'testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38241/hbase 2024-12-01T18:49:45,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741854_1030 (size=64) 2024-12-01T18:49:45,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741854_1030 (size=64) 2024-12-01T18:49:45,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741854_1030 (size=64) 2024-12-01T18:49:45,692 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:45,694 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:45,696 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fde08b92802f1a5199c8495f09f916a0 columnFamilyName a 2024-12-01T18:49:45,696 DEBUG [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:45,696 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] regionserver.HStore(327): Store=fde08b92802f1a5199c8495f09f916a0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:45,696 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:45,698 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fde08b92802f1a5199c8495f09f916a0 columnFamilyName b 2024-12-01T18:49:45,698 DEBUG [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:45,698 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] regionserver.HStore(327): Store=fde08b92802f1a5199c8495f09f916a0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:45,699 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:45,700 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fde08b92802f1a5199c8495f09f916a0 columnFamilyName c 2024-12-01T18:49:45,701 DEBUG [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:45,701 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] regionserver.HStore(327): Store=fde08b92802f1a5199c8495f09f916a0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:45,701 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:45,702 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:45,703 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:45,704 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:45,704 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:45,705 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:49:45,706 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:45,708 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:49:45,709 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fde08b92802f1a5199c8495f09f916a0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72406929, jitterRate=0.07894732058048248}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:49:45,709 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fde08b92802f1a5199c8495f09f916a0: Writing region info on filesystem at 1733078985692Initializing all the Stores at 1733078985693 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078985694 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078985694Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078985694Cleaning up temporary data from old regions at 1733078985704 (+10 ms)Region opened successfully at 1733078985709 (+5 ms) 2024-12-01T18:49:45,709 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fde08b92802f1a5199c8495f09f916a0, disabling compactions & flushes 2024-12-01T18:49:45,709 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0. 2024-12-01T18:49:45,709 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0. 2024-12-01T18:49:45,709 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0. after waiting 0 ms 2024-12-01T18:49:45,710 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0. 2024-12-01T18:49:45,710 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0. 2024-12-01T18:49:45,710 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fde08b92802f1a5199c8495f09f916a0: Waiting for close lock at 1733078985709Disabling compacts and flushes for region at 1733078985709Disabling writes for close at 1733078985709Writing region close event to WAL at 1733078985710 (+1 ms)Closed at 1733078985710 2024-12-01T18:49:45,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741853_1029 (size=175) 2024-12-01T18:49:45,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741853_1029 (size=175) 2024-12-01T18:49:45,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741853_1029 (size=175) 2024-12-01T18:49:45,717 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:49:45,717 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-95754250:(num 1733078985635) 2024-12-01T18:49:45,717 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:49:45,719 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:45,740 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720, exclude list is [], retry=0 2024-12-01T18:49:45,743 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:45,744 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:45,744 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:45,774 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720 2024-12-01T18:49:45,774 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:49:45,953 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720, size=0 (0bytes) 2024-12-01T18:49:45,953 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720 might be still open, length is 0 2024-12-01T18:49:45,953 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720 2024-12-01T18:49:45,954 WARN [IPC Server handler 1 on default port 38241 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-01T18:49:45,955 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720 after 1ms 2024-12-01T18:49:46,066 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:59008 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59008 dst: /127.0.0.1:43787 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43787 remote=/127.0.0.1:59008]. Total timeout mills is 60000, 59855 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:49:46,067 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:36762 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41037:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36762 dst: /127.0.0.1:41037 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:49:46,067 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:52454 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35899:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52454 dst: /127.0.0.1:35899 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:49:46,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741855_1032 (size=497775) 2024-12-01T18:49:46,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741855_1032 (size=497775) 2024-12-01T18:49:46,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741855_1032 (size=497775) 2024-12-01T18:49:49,365 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T18:49:49,411 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-01T18:49:49,956 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720 after 4002ms 2024-12-01T18:49:49,985 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:49,987 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720 took 4034ms 2024-12-01T18:49:49,992 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733078985720.temp 2024-12-01T18:49:50,034 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/0000000000000000001-wal.1733078985720.temp 2024-12-01T18:49:50,080 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720; continuing. 2024-12-01T18:49:50,080 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720 so closing down 2024-12-01T18:49:50,081 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:49:50,081 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:49:50,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741856_1033 (size=497783) 2024-12-01T18:49:50,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741856_1033 (size=497783) 2024-12-01T18:49:50,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741856_1033 (size=497783) 2024-12-01T18:49:50,086 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/0000000000000000001-wal.1733078985720.temp (wrote 3002 edits, skipped 0 edits in 43 ms) 2024-12-01T18:49:50,088 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/0000000000000000001-wal.1733078985720.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/0000000000000003002 2024-12-01T18:49:50,088 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 100 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720, size=0, length=0, corrupted=false, cancelled=false 2024-12-01T18:49:50,089 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720, journal: Splitting hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720, size=0 (0bytes) at 1733078985953Split 1024 edits, skipped 0 edits. at 1733078990033 (+4080 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/0000000000000000001-wal.1733078985720.temp at 1733078990035 (+2 ms)Split 2048 edits, skipped 0 edits. at 1733078990058 (+23 ms)Finishing writing output for hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720 so closing down at 1733078990080 (+22 ms)3 split writer threads finished at 1733078990081 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/0000000000000000001-wal.1733078985720.temp (wrote 3002 edits, skipped 0 edits in 43 ms) at 1733078990086 (+5 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/0000000000000000001-wal.1733078985720.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/0000000000000003002 at 1733078990088 (+2 ms)Processed 3002 edits across 1 Regions in 100 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720, size=0, length=0, corrupted=false, cancelled=false at 1733078990088 2024-12-01T18:49:50,091 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720 to hdfs://localhost:38241/hbase/oldWALs/wal.1733078985720 2024-12-01T18:49:50,092 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/0000000000000003002 2024-12-01T18:49:50,092 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:49:50,095 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:50,108 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078990095, exclude list is [], retry=0 2024-12-01T18:49:50,111 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:50,112 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:50,112 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:50,140 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078990095 2024-12-01T18:49:50,140 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:33601:33601)] 2024-12-01T18:49:50,141 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:50,143 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:50,144 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fde08b92802f1a5199c8495f09f916a0 columnFamilyName a 2024-12-01T18:49:50,144 DEBUG [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:50,145 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] regionserver.HStore(327): Store=fde08b92802f1a5199c8495f09f916a0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:50,145 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:50,146 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fde08b92802f1a5199c8495f09f916a0 columnFamilyName b 2024-12-01T18:49:50,146 DEBUG [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:50,146 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] regionserver.HStore(327): Store=fde08b92802f1a5199c8495f09f916a0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:50,146 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:50,148 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fde08b92802f1a5199c8495f09f916a0 columnFamilyName c 2024-12-01T18:49:50,148 DEBUG [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:50,148 INFO [StoreOpener-fde08b92802f1a5199c8495f09f916a0-1 {}] regionserver.HStore(327): Store=fde08b92802f1a5199c8495f09f916a0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:50,149 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:50,150 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:50,152 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:50,153 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/0000000000000003002 2024-12-01T18:49:50,182 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:50,223 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-01T18:49:50,604 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fde08b92802f1a5199c8495f09f916a0 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-01T18:49:50,646 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/a/6421dd8faf584adab7203fd98675501b is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733078985780/Put/seqid=0 2024-12-01T18:49:50,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741858_1035 (size=50463) 2024-12-01T18:49:50,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741858_1035 (size=50463) 2024-12-01T18:49:50,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741858_1035 (size=50463) 2024-12-01T18:49:50,665 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/a/6421dd8faf584adab7203fd98675501b 2024-12-01T18:49:50,677 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/a/6421dd8faf584adab7203fd98675501b as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/a/6421dd8faf584adab7203fd98675501b 2024-12-01T18:49:50,685 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/a/6421dd8faf584adab7203fd98675501b, entries=754, sequenceid=754, filesize=49.3 K 2024-12-01T18:49:50,686 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for fde08b92802f1a5199c8495f09f916a0 in 82ms, sequenceid=754, compaction requested=false; wal=null 2024-12-01T18:49:50,702 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-01T18:49:50,702 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fde08b92802f1a5199c8495f09f916a0 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-01T18:49:50,726 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/a/c432716730d54622adcfe34148b265e1 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733078985810/Put/seqid=0 2024-12-01T18:49:50,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741859_1036 (size=20072) 2024-12-01T18:49:50,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741859_1036 (size=20072) 2024-12-01T18:49:50,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741859_1036 (size=20072) 2024-12-01T18:49:50,762 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/a/c432716730d54622adcfe34148b265e1 2024-12-01T18:49:50,804 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/b/be32abfbe57b45b08a86836b2f835862 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733078985837/Put/seqid=0 2024-12-01T18:49:50,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741860_1037 (size=35835) 2024-12-01T18:49:50,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741860_1037 (size=35835) 2024-12-01T18:49:50,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741860_1037 (size=35835) 2024-12-01T18:49:50,819 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/b/be32abfbe57b45b08a86836b2f835862 2024-12-01T18:49:50,827 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/a/c432716730d54622adcfe34148b265e1 as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/a/c432716730d54622adcfe34148b265e1 2024-12-01T18:49:50,836 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/a/c432716730d54622adcfe34148b265e1, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-01T18:49:50,838 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/b/be32abfbe57b45b08a86836b2f835862 as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/b/be32abfbe57b45b08a86836b2f835862 2024-12-01T18:49:50,846 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/b/be32abfbe57b45b08a86836b2f835862, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-01T18:49:50,847 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for fde08b92802f1a5199c8495f09f916a0 in 144ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-01T18:49:50,857 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-01T18:49:50,857 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fde08b92802f1a5199c8495f09f916a0 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-01T18:49:50,865 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/b/5022c23346fc47fd8fa6cc788971c087 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733078985852/Put/seqid=0 2024-12-01T18:49:50,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741861_1038 (size=35082) 2024-12-01T18:49:50,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741861_1038 (size=35082) 2024-12-01T18:49:50,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741861_1038 (size=35082) 2024-12-01T18:49:50,877 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/b/5022c23346fc47fd8fa6cc788971c087 2024-12-01T18:49:50,902 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/c/3ae953c5221f4d3c9ef227c7c2fb7afb is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733078985879/Put/seqid=0 2024-12-01T18:49:50,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741862_1039 (size=20825) 2024-12-01T18:49:50,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741862_1039 (size=20825) 2024-12-01T18:49:50,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741862_1039 (size=20825) 2024-12-01T18:49:50,913 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/c/3ae953c5221f4d3c9ef227c7c2fb7afb 2024-12-01T18:49:50,921 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/b/5022c23346fc47fd8fa6cc788971c087 as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/b/5022c23346fc47fd8fa6cc788971c087 2024-12-01T18:49:50,928 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/b/5022c23346fc47fd8fa6cc788971c087, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-01T18:49:50,929 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/c/3ae953c5221f4d3c9ef227c7c2fb7afb as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/c/3ae953c5221f4d3c9ef227c7c2fb7afb 2024-12-01T18:49:50,935 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/c/3ae953c5221f4d3c9ef227c7c2fb7afb, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-01T18:49:50,936 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for fde08b92802f1a5199c8495f09f916a0 in 79ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-01T18:49:50,946 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733078985919/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0. 2024-12-01T18:49:50,949 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/0000000000000003002 2024-12-01T18:49:50,950 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-01T18:49:50,950 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fde08b92802f1a5199c8495f09f916a0 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-01T18:49:50,960 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/c/3b6617ee4d64436ebc59d50f0ba1227b is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733078985886/Put/seqid=0 2024-12-01T18:49:50,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741863_1040 (size=50301) 2024-12-01T18:49:50,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741863_1040 (size=50301) 2024-12-01T18:49:50,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741863_1040 (size=50301) 2024-12-01T18:49:50,973 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/c/3b6617ee4d64436ebc59d50f0ba1227b 2024-12-01T18:49:50,983 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3b6617ee4d64436ebc59d50f0ba1227b 2024-12-01T18:49:50,984 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/.tmp/c/3b6617ee4d64436ebc59d50f0ba1227b as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/c/3b6617ee4d64436ebc59d50f0ba1227b 2024-12-01T18:49:50,991 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3b6617ee4d64436ebc59d50f0ba1227b 2024-12-01T18:49:50,991 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/c/3b6617ee4d64436ebc59d50f0ba1227b, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-01T18:49:50,992 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for fde08b92802f1a5199c8495f09f916a0 in 41ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-01T18:49:50,993 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/0000000000000003002 2024-12-01T18:49:50,994 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:50,994 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:50,995 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-01T18:49:50,997 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fde08b92802f1a5199c8495f09f916a0 2024-12-01T18:49:51,000 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenIntoWAL/fde08b92802f1a5199c8495f09f916a0/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-01T18:49:51,001 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fde08b92802f1a5199c8495f09f916a0; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73150173, jitterRate=0.09002251923084259}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-01T18:49:51,001 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fde08b92802f1a5199c8495f09f916a0: Writing region info on filesystem at 1733078990141Initializing all the Stores at 1733078990142 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078990142Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078990142Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078990142Cleaning up temporary data from old regions at 1733078990994 (+852 ms)Region opened successfully at 1733078991001 (+7 ms) 2024-12-01T18:49:51,055 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fde08b92802f1a5199c8495f09f916a0, disabling compactions & flushes 2024-12-01T18:49:51,055 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0. 2024-12-01T18:49:51,055 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0. 2024-12-01T18:49:51,055 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0. after waiting 0 ms 2024-12-01T18:49:51,055 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0. 2024-12-01T18:49:51,058 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733078985629.fde08b92802f1a5199c8495f09f916a0. 2024-12-01T18:49:51,058 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fde08b92802f1a5199c8495f09f916a0: Waiting for close lock at 1733078991055Disabling compacts and flushes for region at 1733078991055Disabling writes for close at 1733078991055Writing region close event to WAL at 1733078991057 (+2 ms)Closed at 1733078991057 2024-12-01T18:49:51,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741857_1034 (size=175) 2024-12-01T18:49:51,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741857_1034 (size=175) 2024-12-01T18:49:51,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741857_1034 (size=175) 2024-12-01T18:49:51,065 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:49:51,066 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733078990095) 2024-12-01T18:49:51,082 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=399 (was 386) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36499 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1168818415_22 at /127.0.0.1:49480 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:38241 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:38241 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1168818415_22 at /127.0.0.1:52490 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43397 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1168818415_22 at /127.0.0.1:43186 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:43397 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:36499 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=862 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=181 (was 197), ProcessCount=11 (was 11), AvailableMemoryMB=4078 (was 4174) 2024-12-01T18:49:51,097 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=399, OpenFileDescriptor=862, MaxFileDescriptor=1048576, SystemLoadAverage=181, ProcessCount=11, AvailableMemoryMB=4074 2024-12-01T18:49:51,114 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:51,117 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:51,118 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:49:51,120 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-07111333, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/hregion-07111333, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:51,135 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-07111333/hregion-07111333.1733078991121, exclude list is [], retry=0 2024-12-01T18:49:51,138 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:51,138 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:51,138 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:51,167 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-07111333/hregion-07111333.1733078991121 2024-12-01T18:49:51,168 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:33601:33601)] 2024-12-01T18:49:51,168 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 51ab14d8454633c5ca0d4c5339602e24, NAME => 'test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38241/hbase 2024-12-01T18:49:51,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741865_1042 (size=43) 2024-12-01T18:49:51,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741865_1042 (size=43) 2024-12-01T18:49:51,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741865_1042 (size=43) 2024-12-01T18:49:51,183 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:51,185 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:51,186 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51ab14d8454633c5ca0d4c5339602e24 columnFamilyName a 2024-12-01T18:49:51,186 DEBUG [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:51,187 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] regionserver.HStore(327): Store=51ab14d8454633c5ca0d4c5339602e24/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:51,187 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:51,189 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51ab14d8454633c5ca0d4c5339602e24 columnFamilyName b 2024-12-01T18:49:51,189 DEBUG [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:51,189 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] regionserver.HStore(327): Store=51ab14d8454633c5ca0d4c5339602e24/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:51,190 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:51,191 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51ab14d8454633c5ca0d4c5339602e24 columnFamilyName c 2024-12-01T18:49:51,191 DEBUG [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:51,192 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] regionserver.HStore(327): Store=51ab14d8454633c5ca0d4c5339602e24/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:51,192 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:51,193 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:51,193 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:51,195 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:51,195 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:51,195 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:49:51,197 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:51,199 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:49:51,200 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 51ab14d8454633c5ca0d4c5339602e24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62387471, jitterRate=-0.07035423815250397}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:49:51,201 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 51ab14d8454633c5ca0d4c5339602e24: Writing region info on filesystem at 1733078991183Initializing all the Stores at 1733078991184 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078991184Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078991184Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078991184Cleaning up temporary data from old regions at 1733078991195 (+11 ms)Region opened successfully at 1733078991201 (+6 ms) 2024-12-01T18:49:51,201 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 51ab14d8454633c5ca0d4c5339602e24, disabling compactions & flushes 2024-12-01T18:49:51,201 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24. 2024-12-01T18:49:51,202 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24. 2024-12-01T18:49:51,202 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24. after waiting 0 ms 2024-12-01T18:49:51,202 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24. 2024-12-01T18:49:51,202 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24. 2024-12-01T18:49:51,202 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 51ab14d8454633c5ca0d4c5339602e24: Waiting for close lock at 1733078991201Disabling compacts and flushes for region at 1733078991201Disabling writes for close at 1733078991202 (+1 ms)Writing region close event to WAL at 1733078991202Closed at 1733078991202 2024-12-01T18:49:51,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741864_1041 (size=175) 2024-12-01T18:49:51,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741864_1041 (size=175) 2024-12-01T18:49:51,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741864_1041 (size=175) 2024-12-01T18:49:51,209 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:49:51,209 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-07111333:(num 1733078991121) 2024-12-01T18:49:51,209 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:49:51,212 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:51,234 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212, exclude list is [], retry=0 2024-12-01T18:49:51,238 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:51,239 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:51,239 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:51,296 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212 2024-12-01T18:49:51,296 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:49:51,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741866_1043 (size=371388) 2024-12-01T18:49:51,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741866_1043 (size=371388) 2024-12-01T18:49:51,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741866_1043 (size=371388) 2024-12-01T18:49:51,485 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212, size=362.7 K (371388bytes) 2024-12-01T18:49:51,485 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212 2024-12-01T18:49:51,485 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212 after 0ms 2024-12-01T18:49:51,516 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:51,518 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212 took 33ms 2024-12-01T18:49:51,522 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-01T18:49:51,522 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733078991212.temp 2024-12-01T18:49:51,522 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-01T18:49:51,524 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T18:49:51,524 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-01T18:49:51,524 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-01T18:49:51,525 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-01T18:49:51,526 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-01T18:49:51,526 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-01T18:49:51,565 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000000001-wal.1733078991212.temp 2024-12-01T18:49:51,572 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212 so closing down 2024-12-01T18:49:51,572 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:49:51,587 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:49:51,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741867_1044 (size=371388) 2024-12-01T18:49:51,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741867_1044 (size=371388) 2024-12-01T18:49:51,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741867_1044 (size=371388) 2024-12-01T18:49:51,593 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000000001-wal.1733078991212.temp (wrote 3000 edits, skipped 0 edits in 20 ms) 2024-12-01T18:49:51,595 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000000001-wal.1733078991212.temp to hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000003000 2024-12-01T18:49:51,596 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 78 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-01T18:49:51,596 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212, journal: Splitting hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212, size=362.7 K (371388bytes) at 1733078991485Split 1024 edits, skipped 0 edits. at 1733078991538 (+53 ms)Split 2048 edits, skipped 0 edits. at 1733078991559 (+21 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000000001-wal.1733078991212.temp at 1733078991565 (+6 ms)Finishing writing output for hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212 so closing down at 1733078991572 (+7 ms)3 split writer threads finished at 1733078991587 (+15 ms)Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000000001-wal.1733078991212.temp (wrote 3000 edits, skipped 0 edits in 20 ms) at 1733078991593 (+6 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000000001-wal.1733078991212.temp to hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000003000 at 1733078991596 (+3 ms)Processed 3000 edits across 1 Regions in 78 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733078991596 2024-12-01T18:49:51,598 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991212 to hdfs://localhost:38241/hbase/oldWALs/wal.1733078991212 2024-12-01T18:49:51,599 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000003000 2024-12-01T18:49:51,599 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:49:51,601 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:51,621 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602, exclude list is [], retry=0 2024-12-01T18:49:51,624 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:51,625 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:51,625 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:51,677 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602 2024-12-01T18:49:51,681 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601)] 2024-12-01T18:49:51,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741868_1045 (size=371515) 2024-12-01T18:49:51,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741868_1045 (size=371515) 2024-12-01T18:49:51,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741868_1045 (size=371515) 2024-12-01T18:49:51,912 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602, size=362.8 K (371515bytes) 2024-12-01T18:49:51,912 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602 2024-12-01T18:49:51,913 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602 after 1ms 2024-12-01T18:49:51,954 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:51,956 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602 took 44ms 2024-12-01T18:49:51,961 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733078991602.temp 2024-12-01T18:49:52,005 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602 so closing down 2024-12-01T18:49:52,005 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:49:52,015 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000003001-wal.1733078991602.temp 2024-12-01T18:49:52,029 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:49:52,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741869_1046 (size=371515) 2024-12-01T18:49:52,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741869_1046 (size=371515) 2024-12-01T18:49:52,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741869_1046 (size=371515) 2024-12-01T18:49:52,043 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000003001-wal.1733078991602.temp (wrote 3000 edits, skipped 0 edits in 14 ms) 2024-12-01T18:49:52,045 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000003001-wal.1733078991602.temp to hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000006000 2024-12-01T18:49:52,045 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 88 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-01T18:49:52,046 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602, journal: Splitting hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602, size=362.8 K (371515bytes) at 1733078991912Split 1024 edits, skipped 0 edits. at 1733078991970 (+58 ms)Split 2048 edits, skipped 0 edits. at 1733078991982 (+12 ms)Finishing writing output for hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602 so closing down at 1733078992005 (+23 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000003001-wal.1733078991602.temp at 1733078992015 (+10 ms)3 split writer threads finished at 1733078992029 (+14 ms)Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000003001-wal.1733078991602.temp (wrote 3000 edits, skipped 0 edits in 14 ms) at 1733078992043 (+14 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000003001-wal.1733078991602.temp to hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000006000 at 1733078992045 (+2 ms)Processed 3000 edits across 1 Regions in 88 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733078992045 2024-12-01T18:49:52,048 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078991602 to hdfs://localhost:38241/hbase/oldWALs/wal.1733078991602 2024-12-01T18:49:52,049 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000006000 2024-12-01T18:49:52,049 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:49:52,052 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/test2727-manual,16010,1733078991113, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:52,072 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078992052, exclude list is [], retry=0 2024-12-01T18:49:52,075 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:52,075 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:52,076 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:52,112 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733078991113/wal.1733078992052 2024-12-01T18:49:52,113 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:40921:40921)] 2024-12-01T18:49:52,113 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 51ab14d8454633c5ca0d4c5339602e24, NAME => 'test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:52,113 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:52,113 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:52,113 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:52,115 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:52,116 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51ab14d8454633c5ca0d4c5339602e24 columnFamilyName a 2024-12-01T18:49:52,116 DEBUG [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:52,117 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] regionserver.HStore(327): Store=51ab14d8454633c5ca0d4c5339602e24/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:52,117 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:52,117 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51ab14d8454633c5ca0d4c5339602e24 columnFamilyName b 2024-12-01T18:49:52,118 DEBUG [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:52,118 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] regionserver.HStore(327): Store=51ab14d8454633c5ca0d4c5339602e24/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:52,118 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:52,119 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51ab14d8454633c5ca0d4c5339602e24 columnFamilyName c 2024-12-01T18:49:52,119 DEBUG [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:52,120 INFO [StoreOpener-51ab14d8454633c5ca0d4c5339602e24-1 {}] regionserver.HStore(327): Store=51ab14d8454633c5ca0d4c5339602e24/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:52,120 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:52,121 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:52,123 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:52,124 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000003000 2024-12-01T18:49:52,151 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:52,175 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000003000 2024-12-01T18:49:52,176 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000006000 2024-12-01T18:49:52,203 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:52,226 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000006000 2024-12-01T18:49:52,226 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 51ab14d8454633c5ca0d4c5339602e24 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-01T18:49:52,248 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/.tmp/a/af624b9cebd44b1997f6c7099e207bc9 is 41, key is test2727/a:100/1733078991687/Put/seqid=0 2024-12-01T18:49:52,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741871_1048 (size=84227) 2024-12-01T18:49:52,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741871_1048 (size=84227) 2024-12-01T18:49:52,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741871_1048 (size=84227) 2024-12-01T18:49:52,259 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/.tmp/a/af624b9cebd44b1997f6c7099e207bc9 2024-12-01T18:49:52,284 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/.tmp/b/a682e13ba9784bfe9fe1f7b789324c35 is 41, key is test2727/b:100/1733078991758/Put/seqid=0 2024-12-01T18:49:52,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741872_1049 (size=84609) 2024-12-01T18:49:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741872_1049 (size=84609) 2024-12-01T18:49:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741872_1049 (size=84609) 2024-12-01T18:49:52,291 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/.tmp/b/a682e13ba9784bfe9fe1f7b789324c35 2024-12-01T18:49:52,328 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/.tmp/c/a986be46d9ed4cefb82ebb4cd83ba7d7 is 41, key is test2727/c:100/1733078991819/Put/seqid=0 2024-12-01T18:49:52,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741873_1050 (size=84609) 2024-12-01T18:49:52,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741873_1050 (size=84609) 2024-12-01T18:49:52,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741873_1050 (size=84609) 2024-12-01T18:49:52,344 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/.tmp/c/a986be46d9ed4cefb82ebb4cd83ba7d7 2024-12-01T18:49:52,351 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/.tmp/a/af624b9cebd44b1997f6c7099e207bc9 as hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/a/af624b9cebd44b1997f6c7099e207bc9 2024-12-01T18:49:52,358 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/a/af624b9cebd44b1997f6c7099e207bc9, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-01T18:49:52,359 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/.tmp/b/a682e13ba9784bfe9fe1f7b789324c35 as hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/b/a682e13ba9784bfe9fe1f7b789324c35 2024-12-01T18:49:52,366 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/b/a682e13ba9784bfe9fe1f7b789324c35, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-01T18:49:52,367 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/.tmp/c/a986be46d9ed4cefb82ebb4cd83ba7d7 as hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/c/a986be46d9ed4cefb82ebb4cd83ba7d7 2024-12-01T18:49:52,372 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/c/a986be46d9ed4cefb82ebb4cd83ba7d7, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-01T18:49:52,373 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 51ab14d8454633c5ca0d4c5339602e24 in 147ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-01T18:49:52,374 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000003000 2024-12-01T18:49:52,374 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/0000000000000006000 2024-12-01T18:49:52,375 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:52,375 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:52,376 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:49:52,378 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 51ab14d8454633c5ca0d4c5339602e24 2024-12-01T18:49:52,380 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/test2727/51ab14d8454633c5ca0d4c5339602e24/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-01T18:49:52,381 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 51ab14d8454633c5ca0d4c5339602e24; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71935644, jitterRate=0.07192462682723999}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:49:52,382 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 51ab14d8454633c5ca0d4c5339602e24: Writing region info on filesystem at 1733078992113Initializing all the Stores at 1733078992114 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078992114Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078992115 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078992115Obtaining lock to block concurrent updates at 1733078992226 (+111 ms)Preparing flush snapshotting stores in 51ab14d8454633c5ca0d4c5339602e24 at 1733078992226Finished memstore snapshotting test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733078992227 (+1 ms)Flushing stores of test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24. at 1733078992227Flushing 51ab14d8454633c5ca0d4c5339602e24/a: creating writer at 1733078992227Flushing 51ab14d8454633c5ca0d4c5339602e24/a: appending metadata at 1733078992247 (+20 ms)Flushing 51ab14d8454633c5ca0d4c5339602e24/a: closing flushed file at 1733078992247Flushing 51ab14d8454633c5ca0d4c5339602e24/b: creating writer at 1733078992264 (+17 ms)Flushing 51ab14d8454633c5ca0d4c5339602e24/b: appending metadata at 1733078992283 (+19 ms)Flushing 51ab14d8454633c5ca0d4c5339602e24/b: closing flushed file at 1733078992283Flushing 51ab14d8454633c5ca0d4c5339602e24/c: creating writer at 1733078992298 (+15 ms)Flushing 51ab14d8454633c5ca0d4c5339602e24/c: appending metadata at 1733078992327 (+29 ms)Flushing 51ab14d8454633c5ca0d4c5339602e24/c: closing flushed file at 1733078992327Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ca1b9bc: reopening flushed file at 1733078992350 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e249e6e: reopening flushed file at 1733078992358 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@780c209b: reopening flushed file at 1733078992366 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 51ab14d8454633c5ca0d4c5339602e24 in 147ms, sequenceid=6000, compaction requested=false; wal=null at 1733078992373 (+7 ms)Cleaning up temporary data from old regions at 1733078992375 (+2 ms)Region opened successfully at 1733078992382 (+7 ms) 2024-12-01T18:49:52,383 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-01T18:49:52,383 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 51ab14d8454633c5ca0d4c5339602e24, disabling compactions & flushes 2024-12-01T18:49:52,384 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24. 2024-12-01T18:49:52,384 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24. 2024-12-01T18:49:52,384 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24. after waiting 0 ms 2024-12-01T18:49:52,384 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24. 2024-12-01T18:49:52,385 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733078991115.51ab14d8454633c5ca0d4c5339602e24. 2024-12-01T18:49:52,385 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 51ab14d8454633c5ca0d4c5339602e24: Waiting for close lock at 1733078992383Disabling compacts and flushes for region at 1733078992383Disabling writes for close at 1733078992384 (+1 ms)Writing region close event to WAL at 1733078992385 (+1 ms)Closed at 1733078992385 2024-12-01T18:49:52,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741870_1047 (size=175) 2024-12-01T18:49:52,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741870_1047 (size=175) 2024-12-01T18:49:52,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741870_1047 (size=175) 2024-12-01T18:49:52,395 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:49:52,395 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733078992052) 2024-12-01T18:49:52,413 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=402 (was 399) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:40440 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:49554 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:52490 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=925 (was 862) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=215 (was 181) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3855 (was 4074) 2024-12-01T18:49:52,427 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=402, OpenFileDescriptor=925, MaxFileDescriptor=1048576, SystemLoadAverage=215, ProcessCount=11, AvailableMemoryMB=3855 2024-12-01T18:49:52,451 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:52,459 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733078992450, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:52,459 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733078992459 2024-12-01T18:49:52,494 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733078992450/wal.1733078992459 2024-12-01T18:49:52,496 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:49:52,497 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 89a3641c047acb397b4a5f78239028a0, NAME => 'testSequentialEditLogSeqNum,,1733078992452.89a3641c047acb397b4a5f78239028a0.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:52,497 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733078992452.89a3641c047acb397b4a5f78239028a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:52,497 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 89a3641c047acb397b4a5f78239028a0 2024-12-01T18:49:52,497 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 89a3641c047acb397b4a5f78239028a0 2024-12-01T18:49:52,498 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0 doesn't exist for region: 89a3641c047acb397b4a5f78239028a0 on table testSequentialEditLogSeqNum 2024-12-01T18:49:52,499 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 89a3641c047acb397b4a5f78239028a0 on table testSequentialEditLogSeqNum 2024-12-01T18:49:52,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741875_1052 (size=62) 2024-12-01T18:49:52,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741875_1052 (size=62) 2024-12-01T18:49:52,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741875_1052 (size=62) 2024-12-01T18:49:52,510 INFO [StoreOpener-89a3641c047acb397b4a5f78239028a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89a3641c047acb397b4a5f78239028a0 2024-12-01T18:49:52,511 INFO [StoreOpener-89a3641c047acb397b4a5f78239028a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89a3641c047acb397b4a5f78239028a0 columnFamilyName a 2024-12-01T18:49:52,511 DEBUG [StoreOpener-89a3641c047acb397b4a5f78239028a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:52,512 INFO [StoreOpener-89a3641c047acb397b4a5f78239028a0-1 {}] regionserver.HStore(327): Store=89a3641c047acb397b4a5f78239028a0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:52,512 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89a3641c047acb397b4a5f78239028a0 2024-12-01T18:49:52,513 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0 2024-12-01T18:49:52,514 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0 2024-12-01T18:49:52,514 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89a3641c047acb397b4a5f78239028a0 2024-12-01T18:49:52,514 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89a3641c047acb397b4a5f78239028a0 2024-12-01T18:49:52,516 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89a3641c047acb397b4a5f78239028a0 2024-12-01T18:49:52,519 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:49:52,520 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89a3641c047acb397b4a5f78239028a0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59632980, jitterRate=-0.11139935255050659}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T18:49:52,522 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89a3641c047acb397b4a5f78239028a0: Writing region info on filesystem at 1733078992498Initializing all the Stores at 1733078992509 (+11 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078992509Cleaning up temporary data from old regions at 1733078992514 (+5 ms)Region opened successfully at 1733078992522 (+8 ms) 2024-12-01T18:49:52,536 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 89a3641c047acb397b4a5f78239028a0 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-01T18:49:52,567 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/.tmp/a/f4da4dae3c0f4448971ac3421bce7ea2 is 81, key is testSequentialEditLogSeqNum/a:x0/1733078992522/Put/seqid=0 2024-12-01T18:49:52,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741876_1053 (size=5833) 2024-12-01T18:49:52,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741876_1053 (size=5833) 2024-12-01T18:49:52,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741876_1053 (size=5833) 2024-12-01T18:49:52,576 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/.tmp/a/f4da4dae3c0f4448971ac3421bce7ea2 2024-12-01T18:49:52,583 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/.tmp/a/f4da4dae3c0f4448971ac3421bce7ea2 as hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/a/f4da4dae3c0f4448971ac3421bce7ea2 2024-12-01T18:49:52,589 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/a/f4da4dae3c0f4448971ac3421bce7ea2, entries=10, sequenceid=13, filesize=5.7 K 2024-12-01T18:49:52,591 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 89a3641c047acb397b4a5f78239028a0 in 55ms, sequenceid=13, compaction requested=false 2024-12-01T18:49:52,591 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 89a3641c047acb397b4a5f78239028a0: 2024-12-01T18:49:52,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T18:49:52,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T18:49:52,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T18:49:52,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T18:49:52,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T18:49:52,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741874_1051 (size=3256) 2024-12-01T18:49:52,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741874_1051 (size=3256) 2024-12-01T18:49:52,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741874_1051 (size=3256) 2024-12-01T18:49:52,617 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733078992450/wal.1733078992459, size=3.2 K (3256bytes) 2024-12-01T18:49:52,617 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733078992450/wal.1733078992459 2024-12-01T18:49:52,618 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733078992450/wal.1733078992459 after 1ms 2024-12-01T18:49:52,647 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:52,647 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733078992450/wal.1733078992459 took 30ms 2024-12-01T18:49:52,650 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733078992450/wal.1733078992459 so closing down 2024-12-01T18:49:52,650 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:49:52,651 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733078992459.temp 2024-12-01T18:49:52,678 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/recovered.edits/0000000000000000003-wal.1733078992459.temp 2024-12-01T18:49:52,679 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:49:52,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741877_1054 (size=2650) 2024-12-01T18:49:52,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741877_1054 (size=2650) 2024-12-01T18:49:52,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741877_1054 (size=2650) 2024-12-01T18:49:52,687 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/recovered.edits/0000000000000000003-wal.1733078992459.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-01T18:49:52,689 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/recovered.edits/0000000000000000003-wal.1733078992459.temp to hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/recovered.edits/0000000000000000020 2024-12-01T18:49:52,689 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:38241/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733078992450/wal.1733078992459, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-01T18:49:52,689 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733078992450/wal.1733078992459, journal: Splitting hdfs://localhost:38241/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733078992450/wal.1733078992459, size=3.2 K (3256bytes) at 1733078992617Finishing writing output for hdfs://localhost:38241/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733078992450/wal.1733078992459 so closing down at 1733078992650 (+33 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/recovered.edits/0000000000000000003-wal.1733078992459.temp at 1733078992678 (+28 ms)3 split writer threads finished at 1733078992679 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/recovered.edits/0000000000000000003-wal.1733078992459.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733078992687 (+8 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/recovered.edits/0000000000000000003-wal.1733078992459.temp to hdfs://localhost:38241/hbase/data/default/testSequentialEditLogSeqNum/89a3641c047acb397b4a5f78239028a0/recovered.edits/0000000000000000020 at 1733078992689 (+2 ms)Processed 17 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:38241/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733078992450/wal.1733078992459, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733078992689 2024-12-01T18:49:52,704 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=407 (was 402) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:49554 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:52490 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 925) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=215 (was 215), ProcessCount=11 (was 11), AvailableMemoryMB=3850 (was 3855) 2024-12-01T18:49:52,716 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=407, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=215, ProcessCount=11, AvailableMemoryMB=3849 2024-12-01T18:49:52,732 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:52,734 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:52,777 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:49:52,780 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36609241, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/hregion-36609241, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:52,793 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36609241/hregion-36609241.1733078992780, exclude list is [], retry=0 2024-12-01T18:49:52,796 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:52,797 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:52,797 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:52,825 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36609241/hregion-36609241.1733078992780 2024-12-01T18:49:52,825 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921)] 2024-12-01T18:49:52,825 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 670dd8ec8f34ca33811c6741c389b57a, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38241/hbase 2024-12-01T18:49:52,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741879_1056 (size=70) 2024-12-01T18:49:52,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741879_1056 (size=70) 2024-12-01T18:49:52,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741879_1056 (size=70) 2024-12-01T18:49:52,840 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:52,841 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,843 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 670dd8ec8f34ca33811c6741c389b57a columnFamilyName a 2024-12-01T18:49:52,843 DEBUG [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:52,843 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(327): Store=670dd8ec8f34ca33811c6741c389b57a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:52,843 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,845 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 670dd8ec8f34ca33811c6741c389b57a columnFamilyName b 2024-12-01T18:49:52,845 DEBUG [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:52,845 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(327): Store=670dd8ec8f34ca33811c6741c389b57a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:52,845 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,847 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 670dd8ec8f34ca33811c6741c389b57a columnFamilyName c 2024-12-01T18:49:52,847 DEBUG [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:52,847 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(327): Store=670dd8ec8f34ca33811c6741c389b57a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:52,847 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,848 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,848 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,849 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,849 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,850 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:49:52,851 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,854 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:49:52,854 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 670dd8ec8f34ca33811c6741c389b57a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60425595, jitterRate=-0.09958846867084503}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:49:52,855 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 670dd8ec8f34ca33811c6741c389b57a: Writing region info on filesystem at 1733078992840Initializing all the Stores at 1733078992841 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078992841Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078992841Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078992841Cleaning up temporary data from old regions at 1733078992849 (+8 ms)Region opened successfully at 1733078992855 (+6 ms) 2024-12-01T18:49:52,855 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 670dd8ec8f34ca33811c6741c389b57a, disabling compactions & flushes 2024-12-01T18:49:52,855 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a. 2024-12-01T18:49:52,855 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a. 2024-12-01T18:49:52,855 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a. after waiting 0 ms 2024-12-01T18:49:52,856 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a. 2024-12-01T18:49:52,856 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a. 2024-12-01T18:49:52,856 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 670dd8ec8f34ca33811c6741c389b57a: Waiting for close lock at 1733078992855Disabling compacts and flushes for region at 1733078992855Disabling writes for close at 1733078992855Writing region close event to WAL at 1733078992856 (+1 ms)Closed at 1733078992856 2024-12-01T18:49:52,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741878_1055 (size=175) 2024-12-01T18:49:52,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741878_1055 (size=175) 2024-12-01T18:49:52,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741878_1055 (size=175) 2024-12-01T18:49:52,862 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:49:52,862 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36609241:(num 1733078992780) 2024-12-01T18:49:52,862 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:49:52,864 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:52,878 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864, exclude list is [], retry=0 2024-12-01T18:49:52,880 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:52,881 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:52,881 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:52,907 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864 2024-12-01T18:49:52,908 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:33601:33601)] 2024-12-01T18:49:52,908 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 670dd8ec8f34ca33811c6741c389b57a, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:52,908 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:52,908 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,908 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,910 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,911 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 670dd8ec8f34ca33811c6741c389b57a columnFamilyName a 2024-12-01T18:49:52,911 DEBUG [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:52,911 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(327): Store=670dd8ec8f34ca33811c6741c389b57a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:52,911 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,912 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 670dd8ec8f34ca33811c6741c389b57a columnFamilyName b 2024-12-01T18:49:52,912 DEBUG [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:52,913 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(327): Store=670dd8ec8f34ca33811c6741c389b57a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:52,913 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,913 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 670dd8ec8f34ca33811c6741c389b57a columnFamilyName c 2024-12-01T18:49:52,914 DEBUG [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:52,914 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(327): Store=670dd8ec8f34ca33811c6741c389b57a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:52,914 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,915 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,916 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,917 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,917 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,917 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:49:52,919 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:52,920 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 670dd8ec8f34ca33811c6741c389b57a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60149614, jitterRate=-0.1037009060382843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:49:52,920 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 670dd8ec8f34ca33811c6741c389b57a: Writing region info on filesystem at 1733078992908Initializing all the Stores at 1733078992909 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078992909Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078992909Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078992910 (+1 ms)Cleaning up temporary data from old regions at 1733078992917 (+7 ms)Region opened successfully at 1733078992920 (+3 ms) 2024-12-01T18:49:52,924 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733078992923/Put/seqid=0 2024-12-01T18:49:52,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741881_1058 (size=4826) 2024-12-01T18:49:52,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741881_1058 (size=4826) 2024-12-01T18:49:52,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741881_1058 (size=4826) 2024-12-01T18:49:52,933 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38241/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 670dd8ec8f34ca33811c6741c389b57a/a 2024-12-01T18:49:52,941 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-01T18:49:52,941 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-01T18:49:52,941 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 670dd8ec8f34ca33811c6741c389b57a: 2024-12-01T18:49:52,943 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/a/bd3e7efa429f46f0b28731b80dff3df9_SeqId_3_ 2024-12-01T18:49:52,944 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38241/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 670dd8ec8f34ca33811c6741c389b57a/a as hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/a/bd3e7efa429f46f0b28731b80dff3df9_SeqId_3_ - updating store file list. 2024-12-01T18:49:52,949 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for bd3e7efa429f46f0b28731b80dff3df9_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-01T18:49:52,950 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/a/bd3e7efa429f46f0b28731b80dff3df9_SeqId_3_ into 670dd8ec8f34ca33811c6741c389b57a/a 2024-12-01T18:49:52,950 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38241/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 670dd8ec8f34ca33811c6741c389b57a/a (new location: hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/a/bd3e7efa429f46f0b28731b80dff3df9_SeqId_3_) 2024-12-01T18:49:52,990 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864, size=0 (0bytes) 2024-12-01T18:49:52,990 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864 might be still open, length is 0 2024-12-01T18:49:52,990 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864 2024-12-01T18:49:52,991 WARN [IPC Server handler 0 on default port 38241 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-01T18:49:52,991 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864 after 1ms 2024-12-01T18:49:55,060 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:49702 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49702 dst: /127.0.0.1:43787 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43787 remote=/127.0.0.1:49702]. Total timeout mills is 60000, 57897 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:49:55,060 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:40558 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41037:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40558 dst: /127.0.0.1:41037 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:49:55,060 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:43396 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35899:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43396 dst: /127.0.0.1:35899 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:49:55,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741880_1059 (size=606) 2024-12-01T18:49:55,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741880_1059 (size=606) 2024-12-01T18:49:55,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741880_1059 (size=606) 2024-12-01T18:49:56,992 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864 after 4002ms 2024-12-01T18:49:57,023 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:57,023 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864 took 4033ms 2024-12-01T18:49:57,025 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864; continuing. 2024-12-01T18:49:57,025 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864 so closing down 2024-12-01T18:49:57,025 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:49:57,027 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733078992864.temp 2024-12-01T18:49:57,055 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/0000000000000000005-wal.1733078992864.temp 2024-12-01T18:49:57,055 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:49:57,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741882_1060 (size=329) 2024-12-01T18:49:57,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741882_1060 (size=329) 2024-12-01T18:49:57,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741882_1060 (size=329) 2024-12-01T18:49:57,063 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/0000000000000000005-wal.1733078992864.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-01T18:49:57,065 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/0000000000000000005-wal.1733078992864.temp to hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/0000000000000000005 2024-12-01T18:49:57,065 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 42 ms; skipped=1; WAL=hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864, size=0, length=0, corrupted=false, cancelled=false 2024-12-01T18:49:57,065 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864, journal: Splitting hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864, size=0 (0bytes) at 1733078992990Finishing writing output for hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864 so closing down at 1733078997025 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/0000000000000000005-wal.1733078992864.temp at 1733078997055 (+30 ms)3 split writer threads finished at 1733078997055Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/0000000000000000005-wal.1733078992864.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733078997063 (+8 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/0000000000000000005-wal.1733078992864.temp to hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/0000000000000000005 at 1733078997065 (+2 ms)Processed 2 edits across 1 Regions in 42 ms; skipped=1; WAL=hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864, size=0, length=0, corrupted=false, cancelled=false at 1733078997065 2024-12-01T18:49:57,067 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864 to hdfs://localhost:38241/hbase/oldWALs/wal.1733078992864 2024-12-01T18:49:57,068 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/0000000000000000005 2024-12-01T18:49:57,068 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:49:57,069 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:49:57,083 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078997070, exclude list is [], retry=0 2024-12-01T18:49:57,085 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:49:57,086 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:49:57,086 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:49:57,118 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078997070 2024-12-01T18:49:57,118 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601)] 2024-12-01T18:49:57,118 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 670dd8ec8f34ca33811c6741c389b57a, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:57,119 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:57,119 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:57,119 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:57,121 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:57,121 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 670dd8ec8f34ca33811c6741c389b57a columnFamilyName a 2024-12-01T18:49:57,122 DEBUG [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:57,128 DEBUG [StoreFileOpener-670dd8ec8f34ca33811c6741c389b57a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for bd3e7efa429f46f0b28731b80dff3df9_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-01T18:49:57,128 DEBUG [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/a/bd3e7efa429f46f0b28731b80dff3df9_SeqId_3_ 2024-12-01T18:49:57,129 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(327): Store=670dd8ec8f34ca33811c6741c389b57a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:57,129 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:57,130 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 670dd8ec8f34ca33811c6741c389b57a columnFamilyName b 2024-12-01T18:49:57,130 DEBUG [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:57,131 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(327): Store=670dd8ec8f34ca33811c6741c389b57a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:57,131 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:57,132 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 670dd8ec8f34ca33811c6741c389b57a columnFamilyName c 2024-12-01T18:49:57,132 DEBUG [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:57,132 INFO [StoreOpener-670dd8ec8f34ca33811c6741c389b57a-1 {}] regionserver.HStore(327): Store=670dd8ec8f34ca33811c6741c389b57a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:57,132 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:57,133 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:57,135 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:57,135 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/0000000000000000005 2024-12-01T18:49:57,163 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:57,164 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/0000000000000000005 2024-12-01T18:49:57,164 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 670dd8ec8f34ca33811c6741c389b57a 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-01T18:49:57,179 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/.tmp/a/e986920a0a21454daab337a06feedd56 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733078992955/Put/seqid=0 2024-12-01T18:49:57,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741884_1062 (size=5149) 2024-12-01T18:49:57,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741884_1062 (size=5149) 2024-12-01T18:49:57,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741884_1062 (size=5149) 2024-12-01T18:49:57,186 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/.tmp/a/e986920a0a21454daab337a06feedd56 2024-12-01T18:49:57,192 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/.tmp/a/e986920a0a21454daab337a06feedd56 as hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/a/e986920a0a21454daab337a06feedd56 2024-12-01T18:49:57,197 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/a/e986920a0a21454daab337a06feedd56, entries=1, sequenceid=5, filesize=5.0 K 2024-12-01T18:49:57,197 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 670dd8ec8f34ca33811c6741c389b57a in 33ms, sequenceid=5, compaction requested=false; wal=null 2024-12-01T18:49:57,198 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/0000000000000000005 2024-12-01T18:49:57,199 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:57,199 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:57,200 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:49:57,202 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 670dd8ec8f34ca33811c6741c389b57a 2024-12-01T18:49:57,204 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/670dd8ec8f34ca33811c6741c389b57a/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-01T18:49:57,205 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 670dd8ec8f34ca33811c6741c389b57a; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60943097, jitterRate=-0.09187708795070648}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:49:57,205 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 670dd8ec8f34ca33811c6741c389b57a: Writing region info on filesystem at 1733078997119Initializing all the Stores at 1733078997120 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078997120Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078997120Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078997120Obtaining lock to block concurrent updates at 1733078997164 (+44 ms)Preparing flush snapshotting stores in 670dd8ec8f34ca33811c6741c389b57a at 1733078997164Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733078997164Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a. at 1733078997164Flushing 670dd8ec8f34ca33811c6741c389b57a/a: creating writer at 1733078997165 (+1 ms)Flushing 670dd8ec8f34ca33811c6741c389b57a/a: appending metadata at 1733078997178 (+13 ms)Flushing 670dd8ec8f34ca33811c6741c389b57a/a: closing flushed file at 1733078997179 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c4a0f94: reopening flushed file at 1733078997191 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 670dd8ec8f34ca33811c6741c389b57a in 33ms, sequenceid=5, compaction requested=false; wal=null at 1733078997197 (+6 ms)Cleaning up temporary data from old regions at 1733078997199 (+2 ms)Region opened successfully at 1733078997205 (+6 ms) 2024-12-01T18:49:57,210 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 670dd8ec8f34ca33811c6741c389b57a, disabling compactions & flushes 2024-12-01T18:49:57,210 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a. 2024-12-01T18:49:57,210 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a. 2024-12-01T18:49:57,210 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a. after waiting 0 ms 2024-12-01T18:49:57,210 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a. 2024-12-01T18:49:57,211 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733078992733.670dd8ec8f34ca33811c6741c389b57a. 2024-12-01T18:49:57,211 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 670dd8ec8f34ca33811c6741c389b57a: Waiting for close lock at 1733078997210Disabling compacts and flushes for region at 1733078997210Disabling writes for close at 1733078997210Writing region close event to WAL at 1733078997211 (+1 ms)Closed at 1733078997211 2024-12-01T18:49:57,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741883_1061 (size=175) 2024-12-01T18:49:57,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741883_1061 (size=175) 2024-12-01T18:49:57,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741883_1061 (size=175) 2024-12-01T18:49:57,216 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:49:57,216 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733078997070) 2024-12-01T18:49:57,233 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=410 (was 407) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:38241 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:38241 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_386933068_22 at /127.0.0.1:40582 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_386933068_22 at /127.0.0.1:49740 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1021 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=197 (was 215), ProcessCount=11 (was 11), AvailableMemoryMB=3813 (was 3849) 2024-12-01T18:49:57,248 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410, OpenFileDescriptor=1021, MaxFileDescriptor=1048576, SystemLoadAverage=197, ProcessCount=11, AvailableMemoryMB=3812 2024-12-01T18:49:57,264 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:49:57,269 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T18:49:57,274 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ec37ae3355e,40221,1733078981110 2024-12-01T18:49:57,277 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@39a13123 2024-12-01T18:49:57,278 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T18:49:57,280 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-01T18:49:57,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T18:49:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-01T18:49:57,291 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T18:49:57,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-01T18:49:57,294 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:57,296 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T18:49:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T18:49:57,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741885_1063 (size=694) 2024-12-01T18:49:57,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741885_1063 (size=694) 2024-12-01T18:49:57,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741885_1063 (size=694) 2024-12-01T18:49:57,309 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0ff3eaf0745a3ffb1311b6b12692fba4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018 2024-12-01T18:49:57,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741886_1064 (size=77) 2024-12-01T18:49:57,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741886_1064 (size=77) 2024-12-01T18:49:57,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741886_1064 (size=77) 2024-12-01T18:49:57,319 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:57,319 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 0ff3eaf0745a3ffb1311b6b12692fba4, disabling compactions & flushes 2024-12-01T18:49:57,319 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:57,319 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:57,319 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. after waiting 0 ms 2024-12-01T18:49:57,319 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:57,320 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:57,320 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0ff3eaf0745a3ffb1311b6b12692fba4: Waiting for close lock at 1733078997319Disabling compacts and flushes for region at 1733078997319Disabling writes for close at 1733078997319Writing region close event to WAL at 1733078997320 (+1 ms)Closed at 1733078997320 2024-12-01T18:49:57,321 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T18:49:57,327 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733078997322"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733078997322"}]},"ts":"1733078997322"} 2024-12-01T18:49:57,330 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T18:49:57,332 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T18:49:57,334 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733078997332"}]},"ts":"1733078997332"} 2024-12-01T18:49:57,339 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-01T18:49:57,340 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {9ec37ae3355e=0} racks are {/default-rack=0} 2024-12-01T18:49:57,341 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T18:49:57,341 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T18:49:57,341 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-01T18:49:57,341 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T18:49:57,341 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T18:49:57,341 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-01T18:49:57,341 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T18:49:57,341 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T18:49:57,341 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-01T18:49:57,341 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T18:49:57,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, ASSIGN}] 2024-12-01T18:49:57,345 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, ASSIGN 2024-12-01T18:49:57,346 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, ASSIGN; state=OFFLINE, location=9ec37ae3355e,34539,1733078981952; forceNewPlan=false, retain=false 2024-12-01T18:49:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T18:49:57,498 INFO [9ec37ae3355e:40221 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T18:49:57,499 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0ff3eaf0745a3ffb1311b6b12692fba4, regionState=OPENING, regionLocation=9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:57,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, ASSIGN because future has completed 2024-12-01T18:49:57,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,34539,1733078981952}] 2024-12-01T18:49:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T18:49:57,662 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:57,663 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ff3eaf0745a3ffb1311b6b12692fba4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:57,663 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,663 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:57,663 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,663 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,665 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,666 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff3eaf0745a3ffb1311b6b12692fba4 columnFamilyName cf1 2024-12-01T18:49:57,666 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:57,667 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(327): Store=0ff3eaf0745a3ffb1311b6b12692fba4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:57,667 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,668 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff3eaf0745a3ffb1311b6b12692fba4 columnFamilyName cf2 2024-12-01T18:49:57,668 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:57,669 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(327): Store=0ff3eaf0745a3ffb1311b6b12692fba4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:57,669 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,670 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,670 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,671 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,671 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,672 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T18:49:57,673 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,675 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:49:57,676 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0ff3eaf0745a3ffb1311b6b12692fba4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66757053, jitterRate=-0.005242392420768738}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T18:49:57,676 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,677 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0ff3eaf0745a3ffb1311b6b12692fba4: Running coprocessor pre-open hook at 1733078997663Writing region info on filesystem at 1733078997663Initializing all the Stores at 1733078997664 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078997664Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078997665 (+1 ms)Cleaning up temporary data from old regions at 1733078997671 (+6 ms)Running coprocessor post-open hooks at 1733078997676 (+5 ms)Region opened successfully at 1733078997677 (+1 ms) 2024-12-01T18:49:57,679 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., pid=6, masterSystemTime=1733078997656 2024-12-01T18:49:57,681 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:57,682 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:57,683 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0ff3eaf0745a3ffb1311b6b12692fba4, regionState=OPEN, openSeqNum=2, regionLocation=9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:57,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,34539,1733078981952 because future has completed 2024-12-01T18:49:57,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-01T18:49:57,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,34539,1733078981952 in 184 msec 2024-12-01T18:49:57,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-01T18:49:57,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, ASSIGN in 347 msec 2024-12-01T18:49:57,694 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T18:49:57,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733078997695"}]},"ts":"1733078997695"} 2024-12-01T18:49:57,697 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-01T18:49:57,699 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T18:49:57,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 414 msec 2024-12-01T18:49:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T18:49:57,929 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-01T18:49:57,929 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-01T18:49:57,931 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T18:49:57,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-01T18:49:57,937 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T18:49:57,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-01T18:49:57,952 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=2] 2024-12-01T18:49:57,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=0ff3eaf0745a3ffb1311b6b12692fba4, source=9ec37ae3355e,34539,1733078981952, destination=9ec37ae3355e,37405,1733078981806, warming up region on 9ec37ae3355e,37405,1733078981806 2024-12-01T18:49:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T18:49:57,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=0ff3eaf0745a3ffb1311b6b12692fba4, source=9ec37ae3355e,34539,1733078981952, destination=9ec37ae3355e,37405,1733078981806, running balancer 2024-12-01T18:49:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, REOPEN/MOVE 2024-12-01T18:49:57,975 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, REOPEN/MOVE 2024-12-01T18:49:57,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46773, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T18:49:57,978 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0ff3eaf0745a3ffb1311b6b12692fba4, regionState=CLOSING, regionLocation=9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:57,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, REOPEN/MOVE because future has completed 2024-12-01T18:49:57,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37405 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:57,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37405 {}] regionserver.HRegion(7855): Warmup {ENCODED => 0ff3eaf0745a3ffb1311b6b12692fba4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:57,981 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-01T18:49:57,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37405 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:57,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,34539,1733078981952}] 2024-12-01T18:49:57,982 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,983 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff3eaf0745a3ffb1311b6b12692fba4 columnFamilyName cf1 2024-12-01T18:49:57,984 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:57,984 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(327): Store=0ff3eaf0745a3ffb1311b6b12692fba4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:57,984 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:57,985 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff3eaf0745a3ffb1311b6b12692fba4 columnFamilyName cf2 2024-12-01T18:49:57,986 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:57,986 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(327): Store=0ff3eaf0745a3ffb1311b6b12692fba4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37405 {}] regionserver.HRegion(1722): Closing 0ff3eaf0745a3ffb1311b6b12692fba4, disabling compactions & flushes 2024-12-01T18:49:57,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37405 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37405 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37405 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. after waiting 0 ms 2024-12-01T18:49:57,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37405 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:57,987 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37405 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:57,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37405 {}] regionserver.HRegion(1676): Region close journal for 0ff3eaf0745a3ffb1311b6b12692fba4: Waiting for close lock at 1733078997986Disabling compacts and flushes for region at 1733078997986Disabling writes for close at 1733078997987 (+1 ms)Writing region close event to WAL at 1733078997987Closed at 1733078997987 2024-12-01T18:49:58,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-01T18:49:58,141 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,141 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-01T18:49:58,142 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 0ff3eaf0745a3ffb1311b6b12692fba4, disabling compactions & flushes 2024-12-01T18:49:58,142 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,142 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,142 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. after waiting 0 ms 2024-12-01T18:49:58,142 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,142 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 0ff3eaf0745a3ffb1311b6b12692fba4 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-01T18:49:58,163 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp/cf1/11203a9949414325a3561fd54587230f is 35, key is r1/cf1:q/1733078997954/Put/seqid=0 2024-12-01T18:49:58,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741887_1065 (size=4783) 2024-12-01T18:49:58,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741887_1065 (size=4783) 2024-12-01T18:49:58,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741887_1065 (size=4783) 2024-12-01T18:49:58,172 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp/cf1/11203a9949414325a3561fd54587230f 2024-12-01T18:49:58,179 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp/cf1/11203a9949414325a3561fd54587230f as hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/11203a9949414325a3561fd54587230f 2024-12-01T18:49:58,186 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/11203a9949414325a3561fd54587230f, entries=1, sequenceid=5, filesize=4.7 K 2024-12-01T18:49:58,187 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 0ff3eaf0745a3ffb1311b6b12692fba4 in 45ms, sequenceid=5, compaction requested=false 2024-12-01T18:49:58,187 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-01T18:49:58,192 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-01T18:49:58,195 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,195 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 0ff3eaf0745a3ffb1311b6b12692fba4: Waiting for close lock at 1733078998142Running coprocessor pre-close hooks at 1733078998142Disabling compacts and flushes for region at 1733078998142Disabling writes for close at 1733078998142Obtaining lock to block concurrent updates at 1733078998142Preparing flush snapshotting stores in 0ff3eaf0745a3ffb1311b6b12692fba4 at 1733078998142Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733078998142Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. at 1733078998143 (+1 ms)Flushing 0ff3eaf0745a3ffb1311b6b12692fba4/cf1: creating writer at 1733078998143Flushing 0ff3eaf0745a3ffb1311b6b12692fba4/cf1: appending metadata at 1733078998163 (+20 ms)Flushing 0ff3eaf0745a3ffb1311b6b12692fba4/cf1: closing flushed file at 1733078998163Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19015e4e: reopening flushed file at 1733078998179 (+16 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 0ff3eaf0745a3ffb1311b6b12692fba4 in 45ms, sequenceid=5, compaction requested=false at 1733078998187 (+8 ms)Writing region close event to WAL at 1733078998188 (+1 ms)Running coprocessor post-close hooks at 1733078998193 (+5 ms)Closed at 1733078998195 (+2 ms) 2024-12-01T18:49:58,195 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 0ff3eaf0745a3ffb1311b6b12692fba4 move to 9ec37ae3355e,37405,1733078981806 record at close sequenceid=5 2024-12-01T18:49:58,198 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,199 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0ff3eaf0745a3ffb1311b6b12692fba4, regionState=CLOSED 2024-12-01T18:49:58,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,34539,1733078981952 because future has completed 2024-12-01T18:49:58,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-01T18:49:58,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,34539,1733078981952 in 221 msec 2024-12-01T18:49:58,208 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, REOPEN/MOVE; state=CLOSED, location=9ec37ae3355e,37405,1733078981806; forceNewPlan=false, retain=false 2024-12-01T18:49:58,359 INFO [9ec37ae3355e:40221 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T18:49:58,359 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0ff3eaf0745a3ffb1311b6b12692fba4, regionState=OPENING, regionLocation=9ec37ae3355e,37405,1733078981806 2024-12-01T18:49:58,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, REOPEN/MOVE because future has completed 2024-12-01T18:49:58,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,37405,1733078981806}] 2024-12-01T18:49:58,521 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,521 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ff3eaf0745a3ffb1311b6b12692fba4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:58,521 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,521 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:58,521 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,522 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,523 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,524 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff3eaf0745a3ffb1311b6b12692fba4 columnFamilyName cf1 2024-12-01T18:49:58,524 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:58,532 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/11203a9949414325a3561fd54587230f 2024-12-01T18:49:58,532 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(327): Store=0ff3eaf0745a3ffb1311b6b12692fba4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:58,532 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,533 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff3eaf0745a3ffb1311b6b12692fba4 columnFamilyName cf2 2024-12-01T18:49:58,533 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:58,534 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(327): Store=0ff3eaf0745a3ffb1311b6b12692fba4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:58,534 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,535 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,536 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,537 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,537 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,537 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T18:49:58,539 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,540 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 0ff3eaf0745a3ffb1311b6b12692fba4; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64625776, jitterRate=-0.03700089454650879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T18:49:58,540 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,541 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 0ff3eaf0745a3ffb1311b6b12692fba4: Running coprocessor pre-open hook at 1733078998522Writing region info on filesystem at 1733078998522Initializing all the Stores at 1733078998523 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078998523Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078998523Cleaning up temporary data from old regions at 1733078998537 (+14 ms)Running coprocessor post-open hooks at 1733078998540 (+3 ms)Region opened successfully at 1733078998541 (+1 ms) 2024-12-01T18:49:58,542 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., pid=9, masterSystemTime=1733078998516 2024-12-01T18:49:58,544 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,544 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,545 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0ff3eaf0745a3ffb1311b6b12692fba4, regionState=OPEN, openSeqNum=9, regionLocation=9ec37ae3355e,37405,1733078981806 2024-12-01T18:49:58,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,37405,1733078981806 because future has completed 2024-12-01T18:49:58,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-01T18:49:58,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,37405,1733078981806 in 186 msec 2024-12-01T18:49:58,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, REOPEN/MOVE in 578 msec 2024-12-01T18:49:58,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T18:49:58,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T18:49:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34539 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:52352 deadline: 1733079058587, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ec37ae3355e port=37405 startCode=1733078981806. As of locationSeqNum=5. 2024-12-01T18:49:58,612 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ec37ae3355e port=37405 startCode=1733078981806. As of locationSeqNum=5. 2024-12-01T18:49:58,613 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ec37ae3355e port=37405 startCode=1733078981806. As of locationSeqNum=5. 2024-12-01T18:49:58,613 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,37405,1733078981806, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ec37ae3355e port=37405 startCode=1733078981806. As of locationSeqNum=5. 2024-12-01T18:49:58,728 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T18:49:58,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T18:49:58,744 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0ff3eaf0745a3ffb1311b6b12692fba4 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-01T18:49:58,760 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp/cf1/656d21ba8549491787b5813fc95a9a2b is 29, key is r1/cf1:/1733078998731/DeleteFamily/seqid=0 2024-12-01T18:49:58,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741888_1066 (size=4906) 2024-12-01T18:49:58,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741888_1066 (size=4906) 2024-12-01T18:49:58,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741888_1066 (size=4906) 2024-12-01T18:49:58,768 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp/cf1/656d21ba8549491787b5813fc95a9a2b 2024-12-01T18:49:58,774 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 656d21ba8549491787b5813fc95a9a2b 2024-12-01T18:49:58,789 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp/cf2/c8e307f71479425d99eb6714593e2b9f is 29, key is r1/cf2:/1733078998731/DeleteFamily/seqid=0 2024-12-01T18:49:58,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741889_1067 (size=4906) 2024-12-01T18:49:58,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741889_1067 (size=4906) 2024-12-01T18:49:58,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741889_1067 (size=4906) 2024-12-01T18:49:58,797 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp/cf2/c8e307f71479425d99eb6714593e2b9f 2024-12-01T18:49:58,802 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c8e307f71479425d99eb6714593e2b9f 2024-12-01T18:49:58,803 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp/cf1/656d21ba8549491787b5813fc95a9a2b as hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/656d21ba8549491787b5813fc95a9a2b 2024-12-01T18:49:58,809 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 656d21ba8549491787b5813fc95a9a2b 2024-12-01T18:49:58,809 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/656d21ba8549491787b5813fc95a9a2b, entries=1, sequenceid=12, filesize=4.8 K 2024-12-01T18:49:58,810 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp/cf2/c8e307f71479425d99eb6714593e2b9f as hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf2/c8e307f71479425d99eb6714593e2b9f 2024-12-01T18:49:58,815 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c8e307f71479425d99eb6714593e2b9f 2024-12-01T18:49:58,816 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf2/c8e307f71479425d99eb6714593e2b9f, entries=1, sequenceid=12, filesize=4.8 K 2024-12-01T18:49:58,817 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 0ff3eaf0745a3ffb1311b6b12692fba4 in 73ms, sequenceid=12, compaction requested=false 2024-12-01T18:49:58,817 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0ff3eaf0745a3ffb1311b6b12692fba4: 2024-12-01T18:49:58,819 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-01T18:49:58,821 DEBUG [Time-limited test {}] regionserver.HStore(1541): 0ff3eaf0745a3ffb1311b6b12692fba4/cf1 is initiating major compaction (all files) 2024-12-01T18:49:58,821 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T18:49:58,821 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:58,821 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 0ff3eaf0745a3ffb1311b6b12692fba4/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,822 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/11203a9949414325a3561fd54587230f, hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/656d21ba8549491787b5813fc95a9a2b] into tmpdir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp, totalSize=9.5 K 2024-12-01T18:49:58,823 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 11203a9949414325a3561fd54587230f, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733078997954 2024-12-01T18:49:58,823 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 656d21ba8549491787b5813fc95a9a2b, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-01T18:49:58,834 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 0ff3eaf0745a3ffb1311b6b12692fba4#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-01T18:49:58,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741890_1068 (size=4626) 2024-12-01T18:49:58,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741890_1068 (size=4626) 2024-12-01T18:49:58,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741890_1068 (size=4626) 2024-12-01T18:49:58,849 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp/cf1/a934817ed3534ddab35b328689f455e8 as hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/a934817ed3534ddab35b328689f455e8 2024-12-01T18:49:58,862 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 0ff3eaf0745a3ffb1311b6b12692fba4/cf1 of 0ff3eaf0745a3ffb1311b6b12692fba4 into a934817ed3534ddab35b328689f455e8(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-01T18:49:58,862 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 0ff3eaf0745a3ffb1311b6b12692fba4: 2024-12-01T18:49:58,863 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-01T18:49:58,863 DEBUG [Time-limited test {}] regionserver.HStore(1541): 0ff3eaf0745a3ffb1311b6b12692fba4/cf2 is initiating major compaction (all files) 2024-12-01T18:49:58,863 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T18:49:58,863 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T18:49:58,863 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 0ff3eaf0745a3ffb1311b6b12692fba4/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,863 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf2/c8e307f71479425d99eb6714593e2b9f] into tmpdir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp, totalSize=4.8 K 2024-12-01T18:49:58,864 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c8e307f71479425d99eb6714593e2b9f, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-01T18:49:58,869 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 0ff3eaf0745a3ffb1311b6b12692fba4#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-01T18:49:58,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741891_1069 (size=4592) 2024-12-01T18:49:58,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741891_1069 (size=4592) 2024-12-01T18:49:58,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741891_1069 (size=4592) 2024-12-01T18:49:58,885 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/.tmp/cf2/cbb77cf776754f81afb587a342cfeaf4 as hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf2/cbb77cf776754f81afb587a342cfeaf4 2024-12-01T18:49:58,892 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 0ff3eaf0745a3ffb1311b6b12692fba4/cf2 of 0ff3eaf0745a3ffb1311b6b12692fba4 into cbb77cf776754f81afb587a342cfeaf4(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-01T18:49:58,892 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 0ff3eaf0745a3ffb1311b6b12692fba4: 2024-12-01T18:49:58,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=0ff3eaf0745a3ffb1311b6b12692fba4, source=9ec37ae3355e,37405,1733078981806, destination=9ec37ae3355e,34539,1733078981952, warming up region on 9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:58,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=0ff3eaf0745a3ffb1311b6b12692fba4, source=9ec37ae3355e,37405,1733078981806, destination=9ec37ae3355e,34539,1733078981952, running balancer 2024-12-01T18:49:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, REOPEN/MOVE 2024-12-01T18:49:58,898 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, REOPEN/MOVE 2024-12-01T18:49:58,899 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=0ff3eaf0745a3ffb1311b6b12692fba4, regionState=CLOSING, regionLocation=9ec37ae3355e,37405,1733078981806 2024-12-01T18:49:58,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34539 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34539 {}] regionserver.HRegion(7855): Warmup {ENCODED => 0ff3eaf0745a3ffb1311b6b12692fba4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34539 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:58,900 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,901 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff3eaf0745a3ffb1311b6b12692fba4 columnFamilyName cf1 2024-12-01T18:49:58,901 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:58,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, REOPEN/MOVE because future has completed 2024-12-01T18:49:58,902 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-01T18:49:58,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,37405,1733078981806}] 2024-12-01T18:49:58,909 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/11203a9949414325a3561fd54587230f 2024-12-01T18:49:58,914 INFO [StoreFileOpener-0ff3eaf0745a3ffb1311b6b12692fba4-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 656d21ba8549491787b5813fc95a9a2b 2024-12-01T18:49:58,914 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/656d21ba8549491787b5813fc95a9a2b 2024-12-01T18:49:58,919 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/a934817ed3534ddab35b328689f455e8 2024-12-01T18:49:58,920 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(327): Store=0ff3eaf0745a3ffb1311b6b12692fba4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:58,920 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:58,921 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff3eaf0745a3ffb1311b6b12692fba4 columnFamilyName cf2 2024-12-01T18:49:58,921 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:58,927 INFO [StoreFileOpener-0ff3eaf0745a3ffb1311b6b12692fba4-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c8e307f71479425d99eb6714593e2b9f 2024-12-01T18:49:58,927 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf2/c8e307f71479425d99eb6714593e2b9f 2024-12-01T18:49:58,932 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf2/cbb77cf776754f81afb587a342cfeaf4 2024-12-01T18:49:58,932 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(327): Store=0ff3eaf0745a3ffb1311b6b12692fba4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34539 {}] regionserver.HRegion(1722): Closing 0ff3eaf0745a3ffb1311b6b12692fba4, disabling compactions & flushes 2024-12-01T18:49:58,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34539 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34539 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34539 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. after waiting 0 ms 2024-12-01T18:49:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34539 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34539 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34539 {}] regionserver.HRegion(1676): Region close journal for 0ff3eaf0745a3ffb1311b6b12692fba4: Waiting for close lock at 1733078998932Disabling compacts and flushes for region at 1733078998932Disabling writes for close at 1733078998932Writing region close event to WAL at 1733078998934 (+2 ms)Closed at 1733078998934 2024-12-01T18:49:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-01T18:49:59,055 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,055 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-01T18:49:59,055 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 0ff3eaf0745a3ffb1311b6b12692fba4, disabling compactions & flushes 2024-12-01T18:49:59,056 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:59,056 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:59,056 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. after waiting 0 ms 2024-12-01T18:49:59,056 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:59,056 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/11203a9949414325a3561fd54587230f, hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/656d21ba8549491787b5813fc95a9a2b] to archive 2024-12-01T18:49:59,060 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-01T18:49:59,064 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/11203a9949414325a3561fd54587230f to hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/11203a9949414325a3561fd54587230f 2024-12-01T18:49:59,066 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/656d21ba8549491787b5813fc95a9a2b to hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/656d21ba8549491787b5813fc95a9a2b 2024-12-01T18:49:59,078 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf2/c8e307f71479425d99eb6714593e2b9f] to archive 2024-12-01T18:49:59,079 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-01T18:49:59,081 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf2/c8e307f71479425d99eb6714593e2b9f to hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf2/c8e307f71479425d99eb6714593e2b9f 2024-12-01T18:49:59,091 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-01T18:49:59,091 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:59,092 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 0ff3eaf0745a3ffb1311b6b12692fba4: Waiting for close lock at 1733078999055Running coprocessor pre-close hooks at 1733078999055Disabling compacts and flushes for region at 1733078999055Disabling writes for close at 1733078999056 (+1 ms)Writing region close event to WAL at 1733078999083 (+27 ms)Running coprocessor post-close hooks at 1733078999091 (+8 ms)Closed at 1733078999091 2024-12-01T18:49:59,092 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 0ff3eaf0745a3ffb1311b6b12692fba4 move to 9ec37ae3355e,34539,1733078981952 record at close sequenceid=12 2024-12-01T18:49:59,094 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,095 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=0ff3eaf0745a3ffb1311b6b12692fba4, regionState=CLOSED 2024-12-01T18:49:59,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,37405,1733078981806 because future has completed 2024-12-01T18:49:59,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-01T18:49:59,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,37405,1733078981806 in 197 msec 2024-12-01T18:49:59,103 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, REOPEN/MOVE; state=CLOSED, location=9ec37ae3355e,34539,1733078981952; forceNewPlan=false, retain=false 2024-12-01T18:49:59,254 INFO [9ec37ae3355e:40221 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T18:49:59,254 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=0ff3eaf0745a3ffb1311b6b12692fba4, regionState=OPENING, regionLocation=9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:59,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, REOPEN/MOVE because future has completed 2024-12-01T18:49:59,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,34539,1733078981952}] 2024-12-01T18:49:59,414 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:59,415 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ff3eaf0745a3ffb1311b6b12692fba4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:49:59,415 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,415 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:49:59,415 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,415 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,417 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,418 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff3eaf0745a3ffb1311b6b12692fba4 columnFamilyName cf1 2024-12-01T18:49:59,418 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:59,425 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/a934817ed3534ddab35b328689f455e8 2024-12-01T18:49:59,425 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(327): Store=0ff3eaf0745a3ffb1311b6b12692fba4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:59,425 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,426 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff3eaf0745a3ffb1311b6b12692fba4 columnFamilyName cf2 2024-12-01T18:49:59,426 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:49:59,433 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf2/cbb77cf776754f81afb587a342cfeaf4 2024-12-01T18:49:59,433 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(327): Store=0ff3eaf0745a3ffb1311b6b12692fba4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:49:59,433 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,434 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,435 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,436 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,436 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,437 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T18:49:59,438 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,439 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 0ff3eaf0745a3ffb1311b6b12692fba4; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72474018, jitterRate=0.07994702458381653}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T18:49:59,439 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,440 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 0ff3eaf0745a3ffb1311b6b12692fba4: Running coprocessor pre-open hook at 1733078999415Writing region info on filesystem at 1733078999415Initializing all the Stores at 1733078999416 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078999416Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733078999416Cleaning up temporary data from old regions at 1733078999436 (+20 ms)Running coprocessor post-open hooks at 1733078999439 (+3 ms)Region opened successfully at 1733078999440 (+1 ms) 2024-12-01T18:49:59,441 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., pid=12, masterSystemTime=1733078999410 2024-12-01T18:49:59,443 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:59,443 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:59,444 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=0ff3eaf0745a3ffb1311b6b12692fba4, regionState=OPEN, openSeqNum=18, regionLocation=9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:59,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,34539,1733078981952 because future has completed 2024-12-01T18:49:59,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-01T18:49:59,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,34539,1733078981952 in 190 msec 2024-12-01T18:49:59,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, REOPEN/MOVE in 554 msec 2024-12-01T18:49:59,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T18:49:59,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T18:49:59,503 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 9ec37ae3355e,34539,1733078981952: testing ***** 2024-12-01T18:49:59,503 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-01T18:49:59,506 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-01T18:49:59,508 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-01T18:49:59,514 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-01T18:49:59,516 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-01T18:49:59,527 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 477125928 }, "NonHeapMemoryUsage": { "committed": 172556288, "init": 7667712, "max": -1, "used": 169556592 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "9ec37ae3355e", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10264, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 38, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 5, "ProcessCallTime_95th_percentile": 12, "ProcessCallTime_98th_percentile": 38, "ProcessCallTime_99th_percentile": 38, "ProcessCallTime_99.9th_percentile": 38, "ProcessCallTime_TimeRangeCount_0-1": 16, "ProcessCallTime_TimeRangeCount_1-3": 9, "ProcessCallTime_TimeRangeCount_3-10": 4, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 38, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 5, "TotalCallTime_95th_percentile": 12, "TotalCallTime_98th_percentile": 38, "TotalCallTime_99th_percentile": 38, "TotalCallTime_99.9th_percentile": 38, "TotalCallTime_TimeRangeCount_0-1": 15, "TotalCallTime_TimeRangeCount_1-3": 10, "TotalCallTime_TimeRangeCount_3-10": 4, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 160, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5781 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "9ec37ae3355e", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:55741", "tag.serverName": "9ec37ae3355e,35069,1733078981915", "tag.clusterId": "bd982dac-d7cc-4a21-a476-59f23a0409a3", "tag.Context": "regionserver", "tag.Hostname": "9ec37ae3355e", "regionCount": 0, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733078981915, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-01T18:49:59,530 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40221 {}] master.MasterRpcServices(700): 9ec37ae3355e,34539,1733078981952 reported a fatal error: ***** ABORTING region server 9ec37ae3355e,34539,1733078981952: testing ***** 2024-12-01T18:49:59,532 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9ec37ae3355e,34539,1733078981952' ***** 2024-12-01T18:49:59,532 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-01T18:49:59,532 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T18:49:59,533 INFO [RS:2;9ec37ae3355e:34539 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-01T18:49:59,533 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T18:49:59,533 INFO [RS:2;9ec37ae3355e:34539 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-01T18:49:59,533 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(3091): Received CLOSE for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:49:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37405 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:53134 deadline: 1733079059533, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ec37ae3355e port=34539 startCode=1733078981952. As of locationSeqNum=12. 2024-12-01T18:49:59,533 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(956): aborting server 9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:59,533 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T18:49:59,534 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0ff3eaf0745a3ffb1311b6b12692fba4, disabling compactions & flushes 2024-12-01T18:49:59,534 INFO [RS:2;9ec37ae3355e:34539 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;9ec37ae3355e:34539. 2024-12-01T18:49:59,534 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,37405,1733078981806, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,37405,1733078981806, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ec37ae3355e port=34539 startCode=1733078981952. As of locationSeqNum=12. 2024-12-01T18:49:59,534 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:59,534 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,37405,1733078981806, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ec37ae3355e port=34539 startCode=1733078981952. As of locationSeqNum=12. 2024-12-01T18:49:59,534 DEBUG [RS:2;9ec37ae3355e:34539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T18:49:59,534 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:59,534 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,37405,1733078981806, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ec37ae3355e port=34539 startCode=1733078981952. As of locationSeqNum=12. 2024-12-01T18:49:59,534 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. after waiting 0 ms 2024-12-01T18:49:59,534 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:59,534 DEBUG [RS:2;9ec37ae3355e:34539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T18:49:59,534 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T18:49:59,535 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T18:49:59,535 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T18:49:59,535 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T18:49:59,535 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-01T18:49:59,535 DEBUG [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 0ff3eaf0745a3ffb1311b6b12692fba4=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.} 2024-12-01T18:49:59,535 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T18:49:59,535 INFO [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T18:49:59,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ec37ae3355e,34539,1733078981952 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:49:59,535 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T18:49:59,535 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T18:49:59,536 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T18:49:59,536 DEBUG [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(1351): Waiting on 0ff3eaf0745a3ffb1311b6b12692fba4, 1588230740 2024-12-01T18:49:59,537 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ec37ae3355e,34539,1733078981952 aborting 2024-12-01T18:49:59,537 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ec37ae3355e,34539,1733078981952 aborting 2024-12-01T18:49:59,537 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:59,537 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 from cache 2024-12-01T18:49:59,537 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0ff3eaf0745a3ffb1311b6b12692fba4: Waiting for close lock at 1733078999533Running coprocessor pre-close hooks at 1733078999533Disabling compacts and flushes for region at 1733078999533Disabling writes for close at 1733078999534 (+1 ms)Writing region close event to WAL at 1733078999536 (+2 ms)Running coprocessor post-close hooks at 1733078999536Closed at 1733078999537 (+1 ms) 2024-12-01T18:49:59,537 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:49:59,537 ERROR [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-01T18:49:59,537 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T18:49:59,538 INFO [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T18:49:59,538 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733078999535Running coprocessor pre-close hooks at 1733078999535Disabling compacts and flushes for region at 1733078999535Disabling writes for close at 1733078999536 (+1 ms)Writing region close event to WAL at 1733078999537 (+1 ms)Running coprocessor post-close hooks at 1733078999537Closed at 1733078999538 (+1 ms) 2024-12-01T18:49:59,538 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T18:49:59,545 INFO [regionserver/9ec37ae3355e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T18:49:59,648 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T18:49:59,649 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1] 2024-12-01T18:49:59,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ec37ae3355e,34539,1733078981952 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:49:59,651 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ec37ae3355e,34539,1733078981952 aborting 2024-12-01T18:49:59,651 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ec37ae3355e,34539,1733078981952 aborting 2024-12-01T18:49:59,651 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 from cache 2024-12-01T18:49:59,736 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(976): stopping server 9ec37ae3355e,34539,1733078981952; all regions closed. 2024-12-01T18:49:59,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741836_1012 (size=7323) 2024-12-01T18:49:59,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741836_1012 (size=7323) 2024-12-01T18:49:59,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741836_1012 (size=7323) 2024-12-01T18:49:59,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741833_1009 (size=2288) 2024-12-01T18:49:59,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741833_1009 (size=2288) 2024-12-01T18:49:59,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741833_1009 (size=2288) 2024-12-01T18:49:59,743 DEBUG [RS:2;9ec37ae3355e:34539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T18:49:59,743 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T18:49:59,743 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T18:49:59,744 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.ChoreService(370): Chore service for: regionserver/9ec37ae3355e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T18:49:59,744 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T18:49:59,744 INFO [regionserver/9ec37ae3355e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T18:49:59,744 INFO [RS:2;9ec37ae3355e:34539 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34539 2024-12-01T18:49:59,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9ec37ae3355e,34539,1733078981952 2024-12-01T18:49:59,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T18:49:59,753 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T18:49:59,755 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9ec37ae3355e,34539,1733078981952] 2024-12-01T18:49:59,756 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9ec37ae3355e,34539,1733078981952 already deleted, retry=false 2024-12-01T18:49:59,756 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9ec37ae3355e,34539,1733078981952 on 9ec37ae3355e,40221,1733078981110 2024-12-01T18:49:59,761 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9ec37ae3355e,34539,1733078981952, splitWal=true, meta=true 2024-12-01T18:49:59,763 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 9ec37ae3355e,34539,1733078981952 (carryingMeta=true) 9ec37ae3355e,34539,1733078981952/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4c152e79[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-01T18:49:59,763 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9ec37ae3355e,34539,1733078981952, splitWal=true, meta=true 2024-12-01T18:49:59,765 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 9ec37ae3355e,34539,1733078981952, splitWal=true, meta=true, isMeta: true 2024-12-01T18:49:59,766 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting 2024-12-01T18:49:59,767 INFO [PEWorker-5 {}] master.SplitWALManager(105): 9ec37ae3355e,34539,1733078981952 WAL count=1, meta=true 2024-12-01T18:49:59,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta}] 2024-12-01T18:49:59,774 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=9ec37ae3355e,37405,1733078981806 2024-12-01T18:49:59,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta, worker=9ec37ae3355e,37405,1733078981806}] 2024-12-01T18:49:59,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T18:49:59,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34539-0x1016f5c81760003, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T18:49:59,855 INFO [RS:2;9ec37ae3355e:34539 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T18:49:59,856 INFO [RS:2;9ec37ae3355e:34539 {}] regionserver.HRegionServer(1031): Exiting; stopping=9ec37ae3355e,34539,1733078981952; zookeeper connection closed. 2024-12-01T18:49:59,856 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@52f274af {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@52f274af 2024-12-01T18:49:59,858 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T18:49:59,859 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1] 2024-12-01T18:49:59,861 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9ec37ae3355e:34539 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9ec37ae3355e/172.17.0.2:34539 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:49:59,861 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1, error=java.net.ConnectException: Call to address=9ec37ae3355e:34539 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9ec37ae3355e/172.17.0.2:34539 2024-12-01T18:49:59,862 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-01T18:49:59,862 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 from cache 2024-12-01T18:49:59,862 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 9ec37ae3355e:34539 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9ec37ae3355e/172.17.0.2:34539 2024-12-01T18:49:59,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37405 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-01T18:49:59,955 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta, size=7.2 K (7323bytes) 2024-12-01T18:49:59,955 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta 2024-12-01T18:49:59,956 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta after 0ms 2024-12-01T18:49:59,983 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:49:59,983 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta took 28ms 2024-12-01T18:49:59,991 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-01T18:49:59,992 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta so closing down 2024-12-01T18:49:59,992 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:49:59,993 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta.temp 2024-12-01T18:50:00,019 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta.temp 2024-12-01T18:50:00,020 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:50:00,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741892_1070 (size=7071) 2024-12-01T18:50:00,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741892_1070 (size=7071) 2024-12-01T18:50:00,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741892_1070 (size=7071) 2024-12-01T18:50:00,033 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-01T18:50:00,035 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta.temp to hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-01T18:50:00,035 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 51 ms; skipped=1; WAL=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-12-01T18:50:00,035 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta, journal: Splitting hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta, size=7.2 K (7323bytes) at 1733078999955Finishing writing output for hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta so closing down at 1733078999992 (+37 ms)Creating recovered edits writer path=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta.temp at 1733079000019 (+27 ms)3 split writer threads finished at 1733079000020 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733079000033 (+13 ms)Rename recovered edits hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta.temp to hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733079000035 (+2 ms)Processed 16 edits across 1 Regions in 51 ms; skipped=1; WAL=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1733079000035 2024-12-01T18:50:00,035 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta 2024-12-01T18:50:00,036 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-01T18:50:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-01T18:50:00,043 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta to hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/oldWALs 2024-12-01T18:50:00,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-01T18:50:00,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta, worker=9ec37ae3355e,37405,1733078981806 in 267 msec 2024-12-01T18:50:00,047 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=9ec37ae3355e,37405,1733078981806 2024-12-01T18:50:00,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-01T18:50:00,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 9ec37ae3355e%2C34539%2C1733078981952.meta.1733078983793.meta, worker=9ec37ae3355e,37405,1733078981806 in 279 msec 2024-12-01T18:50:00,052 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting dir is empty, no logs to split. 2024-12-01T18:50:00,052 INFO [PEWorker-5 {}] master.SplitWALManager(105): 9ec37ae3355e,34539,1733078981952 WAL count=0, meta=true 2024-12-01T18:50:00,052 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if 9ec37ae3355e,34539,1733078981952 WAL splitting is done? wals=0, meta=true 2024-12-01T18:50:00,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T18:50:00,054 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T18:50:00,055 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-01T18:50:00,167 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T18:50:00,168 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1] 2024-12-01T18:50:00,169 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9ec37ae3355e:34539 this server is in the failed servers list 2024-12-01T18:50:00,169 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9ec37ae3355e:34539 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ec37ae3355e:34539 2024-12-01T18:50:00,169 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ec37ae3355e:34539 2024-12-01T18:50:00,169 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 from cache 2024-12-01T18:50:00,206 DEBUG [9ec37ae3355e:40221 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-01T18:50:00,206 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(204): Hosts are {9ec37ae3355e=0} racks are {/default-rack=0} 2024-12-01T18:50:00,206 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T18:50:00,206 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T18:50:00,206 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T18:50:00,206 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T18:50:00,206 INFO [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T18:50:00,206 INFO [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T18:50:00,206 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T18:50:00,207 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9ec37ae3355e,35069,1733078981915 2024-12-01T18:50:00,208 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ec37ae3355e,35069,1733078981915, state=OPENING 2024-12-01T18:50:00,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T18:50:00,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T18:50:00,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T18:50:00,210 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T18:50:00,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:50:00,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:50:00,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:50:00,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9ec37ae3355e,35069,1733078981915}] 2024-12-01T18:50:00,363 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T18:50:00,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54867, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T18:50:00,370 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T18:50:00,370 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:50:00,371 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T18:50:00,372 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ec37ae3355e%2C35069%2C1733078981915.meta, suffix=.meta, logDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,35069,1733078981915, archiveDir=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/oldWALs, maxLogs=32 2024-12-01T18:50:00,393 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,35069,1733078981915/9ec37ae3355e%2C35069%2C1733078981915.meta.1733079000373.meta, exclude list is [], retry=0 2024-12-01T18:50:00,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:00,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:00,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:00,428 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,35069,1733078981915/9ec37ae3355e%2C35069%2C1733078981915.meta.1733079000373.meta 2024-12-01T18:50:00,428 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:50:00,428 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:00,429 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T18:50:00,429 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T18:50:00,429 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T18:50:00,429 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T18:50:00,429 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:00,429 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T18:50:00,429 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T18:50:00,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-01T18:50:00,432 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T18:50:00,432 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:00,432 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:50:00,432 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-01T18:50:00,433 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T18:50:00,433 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:00,433 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:50:00,434 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-01T18:50:00,434 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T18:50:00,434 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:00,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:50:00,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-01T18:50:00,435 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T18:50:00,435 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:00,436 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T18:50:00,436 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T18:50:00,437 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740 2024-12-01T18:50:00,438 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740 2024-12-01T18:50:00,439 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-01T18:50:00,466 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:00,469 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-01T18:50:00,469 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-01T18:50:00,495 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/.tmp/info/991a70b5a2584d3f8f4fd7dad78c4c59 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4./info:regioninfo/1733078999444/Put/seqid=0 2024-12-01T18:50:00,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741894_1072 (size=11177) 2024-12-01T18:50:00,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741894_1072 (size=11177) 2024-12-01T18:50:00,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741894_1072 (size=11177) 2024-12-01T18:50:00,504 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/.tmp/info/991a70b5a2584d3f8f4fd7dad78c4c59 2024-12-01T18:50:00,533 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/.tmp/ns/d1fde1928e0f48519748e9b51c14b1c9 is 43, key is default/ns:d/1733078983988/Put/seqid=0 2024-12-01T18:50:00,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741895_1073 (size=5153) 2024-12-01T18:50:00,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741895_1073 (size=5153) 2024-12-01T18:50:00,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741895_1073 (size=5153) 2024-12-01T18:50:00,541 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/.tmp/ns/d1fde1928e0f48519748e9b51c14b1c9 2024-12-01T18:50:00,562 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/.tmp/table/e4f8a56673b3415fae4db999f2f9121c is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733078997695/Put/seqid=0 2024-12-01T18:50:00,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741896_1074 (size=5431) 2024-12-01T18:50:00,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741896_1074 (size=5431) 2024-12-01T18:50:00,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741896_1074 (size=5431) 2024-12-01T18:50:00,570 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/.tmp/table/e4f8a56673b3415fae4db999f2f9121c 2024-12-01T18:50:00,576 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/.tmp/info/991a70b5a2584d3f8f4fd7dad78c4c59 as hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/info/991a70b5a2584d3f8f4fd7dad78c4c59 2024-12-01T18:50:00,581 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/info/991a70b5a2584d3f8f4fd7dad78c4c59, entries=36, sequenceid=18, filesize=10.9 K 2024-12-01T18:50:00,582 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/.tmp/ns/d1fde1928e0f48519748e9b51c14b1c9 as hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/ns/d1fde1928e0f48519748e9b51c14b1c9 2024-12-01T18:50:00,587 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/ns/d1fde1928e0f48519748e9b51c14b1c9, entries=2, sequenceid=18, filesize=5.0 K 2024-12-01T18:50:00,588 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/.tmp/table/e4f8a56673b3415fae4db999f2f9121c as hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/table/e4f8a56673b3415fae4db999f2f9121c 2024-12-01T18:50:00,593 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/table/e4f8a56673b3415fae4db999f2f9121c, entries=2, sequenceid=18, filesize=5.3 K 2024-12-01T18:50:00,593 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 124ms, sequenceid=18, compaction requested=false; wal=null 2024-12-01T18:50:00,594 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-01T18:50:00,595 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T18:50:00,595 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T18:50:00,596 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T18:50:00,597 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T18:50:00,599 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-01T18:50:00,600 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69204594, jitterRate=0.03122881054878235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T18:50:00,600 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T18:50:00,601 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733079000429Writing region info on filesystem at 1733079000429Initializing all the Stores at 1733079000430 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733079000430Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733079000430Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079000430Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733079000431 (+1 ms)Obtaining lock to block concurrent updates at 1733079000469 (+38 ms)Preparing flush snapshotting stores in 1588230740 at 1733079000469Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733079000470 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733079000470Flushing 1588230740/info: creating writer at 1733079000470Flushing 1588230740/info: appending metadata at 1733079000495 (+25 ms)Flushing 1588230740/info: closing flushed file at 1733079000495Flushing 1588230740/ns: creating writer at 1733079000511 (+16 ms)Flushing 1588230740/ns: appending metadata at 1733079000532 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1733079000532Flushing 1588230740/table: creating writer at 1733079000547 (+15 ms)Flushing 1588230740/table: appending metadata at 1733079000562 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733079000562Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e0a89fa: reopening flushed file at 1733079000575 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d713dea: reopening flushed file at 1733079000581 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f3aaac8: reopening flushed file at 1733079000587 (+6 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 124ms, sequenceid=18, compaction requested=false; wal=null at 1733079000593 (+6 ms)Cleaning up temporary data from old regions at 1733079000595 (+2 ms)Running coprocessor post-open hooks at 1733079000600 (+5 ms)Region opened successfully at 1733079000601 (+1 ms) 2024-12-01T18:50:00,602 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733079000363 2024-12-01T18:50:00,605 DEBUG [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T18:50:00,605 INFO [RS_OPEN_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T18:50:00,606 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=9ec37ae3355e,35069,1733078981915 2024-12-01T18:50:00,607 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ec37ae3355e,35069,1733078981915, state=OPEN 2024-12-01T18:50:00,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T18:50:00,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T18:50:00,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T18:50:00,610 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9ec37ae3355e,35069,1733078981915 2024-12-01T18:50:00,610 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:50:00,610 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:50:00,610 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T18:50:00,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-01T18:50:00,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9ec37ae3355e,35069,1733078981915 in 400 msec 2024-12-01T18:50:00,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-01T18:50:00,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 560 msec 2024-12-01T18:50:00,616 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 9ec37ae3355e,34539,1733078981952 had 2 regions 2024-12-01T18:50:00,617 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 9ec37ae3355e,34539,1733078981952, splitWal=true, meta=true, isMeta: false 2024-12-01T18:50:00,619 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9ec37ae3355e,34539,1733078981952 WAL count=1, meta=false 2024-12-01T18:50:00,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9ec37ae3355e%2C34539%2C1733078981952.1733078983296}] 2024-12-01T18:50:00,621 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=9ec37ae3355e,37405,1733078981806 2024-12-01T18:50:00,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9ec37ae3355e%2C34539%2C1733078981952.1733078983296, worker=9ec37ae3355e,37405,1733078981806}] 2024-12-01T18:50:00,677 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T18:50:00,678 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,35069,1733078981915, seqNum=-1] 2024-12-01T18:50:00,679 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T18:50:00,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T18:50:00,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37405 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-01T18:50:00,791 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.1733078983296, size=2.2 K (2288bytes) 2024-12-01T18:50:00,791 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.1733078983296 2024-12-01T18:50:00,792 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.1733078983296 after 1ms 2024-12-01T18:50:00,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9ec37ae3355e:34539 this server is in the failed servers list 2024-12-01T18:50:00,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9ec37ae3355e:34539 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ec37ae3355e:34539 2024-12-01T18:50:00,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ec37ae3355e:34539 2024-12-01T18:50:00,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=12 from cache 2024-12-01T18:50:00,819 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:00,819 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.1733078983296 took 28ms 2024-12-01T18:50:00,822 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 0ff3eaf0745a3ffb1311b6b12692fba4: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-01T18:50:00,822 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.1733078983296 so closing down 2024-12-01T18:50:00,822 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:50:00,822 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:50:00,823 INFO [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.1733078983296, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-01T18:50:00,823 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.1733078983296, journal: Splitting hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.1733078983296, size=2.2 K (2288bytes) at 1733079000791Finishing writing output for hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.1733078983296 so closing down at 1733079000822 (+31 ms)3 split writer threads finished at 1733079000823 (+1 ms)Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.1733078983296, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733079000823 2024-12-01T18:50:00,823 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.1733078983296 2024-12-01T18:50:00,823 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ec37ae3355e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-01T18:50:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40221 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-01T18:50:00,827 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting/9ec37ae3355e%2C34539%2C1733078981952.1733078983296 to hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/oldWALs 2024-12-01T18:50:00,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-01T18:50:00,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9ec37ae3355e%2C34539%2C1733078981952.1733078983296, worker=9ec37ae3355e,37405,1733078981806 in 205 msec 2024-12-01T18:50:00,831 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=9ec37ae3355e,37405,1733078981806 2024-12-01T18:50:00,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-01T18:50:00,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 9ec37ae3355e%2C34539%2C1733078981952.1733078983296, worker=9ec37ae3355e,37405,1733078981806 in 212 msec 2024-12-01T18:50:00,835 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/WALs/9ec37ae3355e,34539,1733078981952-splitting dir is empty, no logs to split. 2024-12-01T18:50:00,835 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9ec37ae3355e,34539,1733078981952 WAL count=0, meta=false 2024-12-01T18:50:00,835 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 9ec37ae3355e,34539,1733078981952 WAL splitting is done? wals=0, meta=false 2024-12-01T18:50:00,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, ASSIGN}] 2024-12-01T18:50:00,838 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, ASSIGN 2024-12-01T18:50:00,839 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-01T18:50:00,990 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(204): Hosts are {9ec37ae3355e=0} racks are {/default-rack=0} 2024-12-01T18:50:00,990 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T18:50:00,990 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T18:50:00,990 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T18:50:00,990 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T18:50:00,990 INFO [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T18:50:00,990 INFO [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T18:50:00,990 DEBUG [9ec37ae3355e:40221 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T18:50:00,991 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=0ff3eaf0745a3ffb1311b6b12692fba4, regionState=OPENING, regionLocation=9ec37ae3355e,37405,1733078981806 2024-12-01T18:50:00,992 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9ec37ae3355e:34539 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9ec37ae3355e/172.17.0.2:34539 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:00,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1, error=java.net.ConnectException: Call to address=9ec37ae3355e:34539 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9ec37ae3355e/172.17.0.2:34539 2024-12-01T18:50:00,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-01T18:50:00,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,34539,1733078981952, seqNum=-1 from cache 2024-12-01T18:50:00,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 9ec37ae3355e:34539 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9ec37ae3355e/172.17.0.2:34539 2024-12-01T18:50:01,014 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=18] 2024-12-01T18:50:01,014 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9ec37ae3355e:34539 this server is in the failed servers list 2024-12-01T18:50:01,015 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9ec37ae3355e:34539 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ec37ae3355e:34539 2024-12-01T18:50:01,015 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ec37ae3355e:34539 2024-12-01T18:50:01,015 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,34539,1733078981952, seqNum=18 from cache 2024-12-01T18:50:01,098 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T18:50:01,098 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ec37ae3355e,35069,1733078981915, seqNum=-1] 2024-12-01T18:50:01,099 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T18:50:01,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42645, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T18:50:01,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, ASSIGN because future has completed 2024-12-01T18:50:01,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,37405,1733078981806}] 2024-12-01T18:50:01,261 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:50:01,261 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ff3eaf0745a3ffb1311b6b12692fba4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:01,262 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:01,262 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:01,262 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:01,262 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:01,264 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:01,265 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff3eaf0745a3ffb1311b6b12692fba4 columnFamilyName cf1 2024-12-01T18:50:01,265 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:01,271 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf1/a934817ed3534ddab35b328689f455e8 2024-12-01T18:50:01,271 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(327): Store=0ff3eaf0745a3ffb1311b6b12692fba4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:01,272 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:01,272 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff3eaf0745a3ffb1311b6b12692fba4 columnFamilyName cf2 2024-12-01T18:50:01,273 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:01,279 DEBUG [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/cf2/cbb77cf776754f81afb587a342cfeaf4 2024-12-01T18:50:01,279 INFO [StoreOpener-0ff3eaf0745a3ffb1311b6b12692fba4-1 {}] regionserver.HStore(327): Store=0ff3eaf0745a3ffb1311b6b12692fba4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:01,279 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:01,280 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:01,281 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:01,282 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:01,282 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:01,282 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T18:50:01,284 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:01,285 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 0ff3eaf0745a3ffb1311b6b12692fba4; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61532785, jitterRate=-0.08309005200862885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T18:50:01,285 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:01,286 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 0ff3eaf0745a3ffb1311b6b12692fba4: Running coprocessor pre-open hook at 1733079001263Writing region info on filesystem at 1733079001263Initializing all the Stores at 1733079001263Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079001263Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079001264 (+1 ms)Cleaning up temporary data from old regions at 1733079001282 (+18 ms)Running coprocessor post-open hooks at 1733079001285 (+3 ms)Region opened successfully at 1733079001286 (+1 ms) 2024-12-01T18:50:01,287 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., pid=21, masterSystemTime=1733079001257 2024-12-01T18:50:01,290 DEBUG [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:50:01,290 INFO [RS_OPEN_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:50:01,290 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=0ff3eaf0745a3ffb1311b6b12692fba4, regionState=OPEN, openSeqNum=18, regionLocation=9ec37ae3355e,37405,1733078981806 2024-12-01T18:50:01,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,37405,1733078981806 because future has completed 2024-12-01T18:50:01,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-01T18:50:01,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ff3eaf0745a3ffb1311b6b12692fba4, server=9ec37ae3355e,37405,1733078981806 in 190 msec 2024-12-01T18:50:01,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-01T18:50:01,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0ff3eaf0745a3ffb1311b6b12692fba4, ASSIGN in 460 msec 2024-12-01T18:50:01,299 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 9ec37ae3355e,34539,1733078981952 after splitting done 2024-12-01T18:50:01,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 9ec37ae3355e,34539,1733078981952, splitWal=true, meta=true in 1.5420 sec 2024-12-01T18:50:01,323 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4., hostname=9ec37ae3355e,37405,1733078981806, seqNum=18] 2024-12-01T18:50:01,342 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410 (was 410), OpenFileDescriptor=1048 (was 1021) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=197 (was 197), ProcessCount=11 (was 11), AvailableMemoryMB=3758 (was 3812) 2024-12-01T18:50:01,344 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-01T18:50:01,357 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=410, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=197, ProcessCount=11, AvailableMemoryMB=3757 2024-12-01T18:50:01,357 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-01T18:50:01,373 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:50:01,374 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:50:01,375 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:50:01,377 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-17805116, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/hregion-17805116, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:01,390 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-17805116/hregion-17805116.1733079001378, exclude list is [], retry=0 2024-12-01T18:50:01,393 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:01,393 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:01,394 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:01,414 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-01T18:50:01,423 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-17805116/hregion-17805116.1733079001378 2024-12-01T18:50:01,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601)] 2024-12-01T18:50:01,423 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ee5181767944ecd8f2df89e7934875fa, NAME => 'testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38241/hbase 2024-12-01T18:50:01,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741898_1076 (size=67) 2024-12-01T18:50:01,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741898_1076 (size=67) 2024-12-01T18:50:01,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741898_1076 (size=67) 2024-12-01T18:50:01,434 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:01,435 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,437 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5181767944ecd8f2df89e7934875fa columnFamilyName a 2024-12-01T18:50:01,437 DEBUG [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:01,437 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(327): Store=ee5181767944ecd8f2df89e7934875fa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:01,438 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,439 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5181767944ecd8f2df89e7934875fa columnFamilyName b 2024-12-01T18:50:01,439 DEBUG [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:01,439 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(327): Store=ee5181767944ecd8f2df89e7934875fa/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:01,439 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,441 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5181767944ecd8f2df89e7934875fa columnFamilyName c 2024-12-01T18:50:01,441 DEBUG [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:01,441 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(327): Store=ee5181767944ecd8f2df89e7934875fa/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:01,443 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,444 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,444 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,445 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,445 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,446 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:01,447 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,449 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:50:01,449 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee5181767944ecd8f2df89e7934875fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58771630, jitterRate=-0.12423446774482727}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:01,450 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee5181767944ecd8f2df89e7934875fa: Writing region info on filesystem at 1733079001434Initializing all the Stores at 1733079001435 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079001435Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079001435Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079001435Cleaning up temporary data from old regions at 1733079001445 (+10 ms)Region opened successfully at 1733079001450 (+5 ms) 2024-12-01T18:50:01,450 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee5181767944ecd8f2df89e7934875fa, disabling compactions & flushes 2024-12-01T18:50:01,450 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa. 2024-12-01T18:50:01,450 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa. 2024-12-01T18:50:01,450 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa. after waiting 0 ms 2024-12-01T18:50:01,450 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa. 2024-12-01T18:50:01,451 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa. 2024-12-01T18:50:01,451 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee5181767944ecd8f2df89e7934875fa: Waiting for close lock at 1733079001450Disabling compacts and flushes for region at 1733079001450Disabling writes for close at 1733079001450Writing region close event to WAL at 1733079001451 (+1 ms)Closed at 1733079001451 2024-12-01T18:50:01,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741897_1075 (size=175) 2024-12-01T18:50:01,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741897_1075 (size=175) 2024-12-01T18:50:01,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741897_1075 (size=175) 2024-12-01T18:50:01,458 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:50:01,458 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-17805116:(num 1733079001378) 2024-12-01T18:50:01,458 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:50:01,460 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:01,475 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460, exclude list is [], retry=0 2024-12-01T18:50:01,477 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:01,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:01,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:01,510 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460 2024-12-01T18:50:01,511 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601)] 2024-12-01T18:50:01,511 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee5181767944ecd8f2df89e7934875fa, NAME => 'testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:01,511 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:01,511 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,511 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,513 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,514 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5181767944ecd8f2df89e7934875fa columnFamilyName a 2024-12-01T18:50:01,515 DEBUG [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:01,515 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(327): Store=ee5181767944ecd8f2df89e7934875fa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:01,515 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,516 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5181767944ecd8f2df89e7934875fa columnFamilyName b 2024-12-01T18:50:01,516 DEBUG [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:01,517 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(327): Store=ee5181767944ecd8f2df89e7934875fa/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:01,517 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,518 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5181767944ecd8f2df89e7934875fa columnFamilyName c 2024-12-01T18:50:01,518 DEBUG [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:01,518 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(327): Store=ee5181767944ecd8f2df89e7934875fa/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:01,518 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,520 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,521 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-01T18:50:01,521 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-01T18:50:01,521 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,521 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,522 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-01T18:50:01,522 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-01T18:50:01,523 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:01,524 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-01T18:50:01,524 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-01T18:50:01,524 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,525 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee5181767944ecd8f2df89e7934875fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60844809, jitterRate=-0.09334169328212738}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:01,526 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee5181767944ecd8f2df89e7934875fa: Writing region info on filesystem at 1733079001512Initializing all the Stores at 1733079001513 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079001513Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079001513Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079001513Cleaning up temporary data from old regions at 1733079001521 (+8 ms)Region opened successfully at 1733079001526 (+5 ms) 2024-12-01T18:50:01,553 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ee5181767944ecd8f2df89e7934875fa 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-01T18:50:01,570 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/.tmp/a/d41a53f842e74652bcf1ed9edd6fee95 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733079001526/Put/seqid=0 2024-12-01T18:50:01,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741900_1078 (size=5958) 2024-12-01T18:50:01,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741900_1078 (size=5958) 2024-12-01T18:50:01,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741900_1078 (size=5958) 2024-12-01T18:50:01,580 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/.tmp/a/d41a53f842e74652bcf1ed9edd6fee95 2024-12-01T18:50:01,600 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/.tmp/b/90e6d52ff96b4a2ba4061992fe424a73 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733079001537/Put/seqid=0 2024-12-01T18:50:01,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741901_1079 (size=5958) 2024-12-01T18:50:01,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741901_1079 (size=5958) 2024-12-01T18:50:01,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741901_1079 (size=5958) 2024-12-01T18:50:01,609 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/.tmp/b/90e6d52ff96b4a2ba4061992fe424a73 2024-12-01T18:50:01,629 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/.tmp/c/fa0f840cb88d45fca4d9fb1fe7e33df2 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733079001545/Put/seqid=0 2024-12-01T18:50:01,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741902_1080 (size=5958) 2024-12-01T18:50:01,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741902_1080 (size=5958) 2024-12-01T18:50:01,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741902_1080 (size=5958) 2024-12-01T18:50:01,637 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/.tmp/c/fa0f840cb88d45fca4d9fb1fe7e33df2 2024-12-01T18:50:01,644 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/.tmp/a/d41a53f842e74652bcf1ed9edd6fee95 as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/a/d41a53f842e74652bcf1ed9edd6fee95 2024-12-01T18:50:01,650 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/a/d41a53f842e74652bcf1ed9edd6fee95, entries=10, sequenceid=33, filesize=5.8 K 2024-12-01T18:50:01,651 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/.tmp/b/90e6d52ff96b4a2ba4061992fe424a73 as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/b/90e6d52ff96b4a2ba4061992fe424a73 2024-12-01T18:50:01,657 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/b/90e6d52ff96b4a2ba4061992fe424a73, entries=10, sequenceid=33, filesize=5.8 K 2024-12-01T18:50:01,658 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/.tmp/c/fa0f840cb88d45fca4d9fb1fe7e33df2 as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/c/fa0f840cb88d45fca4d9fb1fe7e33df2 2024-12-01T18:50:01,663 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/c/fa0f840cb88d45fca4d9fb1fe7e33df2, entries=10, sequenceid=33, filesize=5.8 K 2024-12-01T18:50:01,664 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for ee5181767944ecd8f2df89e7934875fa in 111ms, sequenceid=33, compaction requested=false 2024-12-01T18:50:01,664 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ee5181767944ecd8f2df89e7934875fa: 2024-12-01T18:50:01,664 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee5181767944ecd8f2df89e7934875fa, disabling compactions & flushes 2024-12-01T18:50:01,664 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa. 2024-12-01T18:50:01,665 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa. 2024-12-01T18:50:01,665 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa. after waiting 0 ms 2024-12-01T18:50:01,665 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa. 2024-12-01T18:50:01,666 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa. 2024-12-01T18:50:01,666 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee5181767944ecd8f2df89e7934875fa: Waiting for close lock at 1733079001664Disabling compacts and flushes for region at 1733079001664Disabling writes for close at 1733079001665 (+1 ms)Writing region close event to WAL at 1733079001666 (+1 ms)Closed at 1733079001666 2024-12-01T18:50:01,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741899_1077 (size=6311) 2024-12-01T18:50:01,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741899_1077 (size=6311) 2024-12-01T18:50:01,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741899_1077 (size=6311) 2024-12-01T18:50:01,675 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/b/90e6d52ff96b4a2ba4061992fe424a73 to hdfs://localhost:38241/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/b/90e6d52ff96b4a2ba4061992fe424a73 2024-12-01T18:50:01,699 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460, size=6.2 K (6311bytes) 2024-12-01T18:50:01,699 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460 2024-12-01T18:50:01,699 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460 after 0ms 2024-12-01T18:50:01,743 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:01,744 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460 took 46ms 2024-12-01T18:50:01,747 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460 so closing down 2024-12-01T18:50:01,747 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:50:01,748 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733079001460.temp 2024-12-01T18:50:01,776 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/0000000000000000003-wal.1733079001460.temp 2024-12-01T18:50:01,777 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:50:01,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741903_1081 (size=5575) 2024-12-01T18:50:01,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741903_1081 (size=5575) 2024-12-01T18:50:01,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741903_1081 (size=5575) 2024-12-01T18:50:01,785 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/0000000000000000003-wal.1733079001460.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-01T18:50:01,786 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/0000000000000000003-wal.1733079001460.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/0000000000000000032 2024-12-01T18:50:01,787 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 43 ms; skipped=2; WAL=hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-01T18:50:01,787 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460, journal: Splitting hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460, size=6.2 K (6311bytes) at 1733079001699Finishing writing output for hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460 so closing down at 1733079001747 (+48 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/0000000000000000003-wal.1733079001460.temp at 1733079001776 (+29 ms)3 split writer threads finished at 1733079001777 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/0000000000000000003-wal.1733079001460.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733079001785 (+8 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/0000000000000000003-wal.1733079001460.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/0000000000000000032 at 1733079001787 (+2 ms)Processed 32 edits across 1 Regions in 43 ms; skipped=2; WAL=hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733079001787 2024-12-01T18:50:01,788 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001460 to hdfs://localhost:38241/hbase/oldWALs/wal.1733079001460 2024-12-01T18:50:01,789 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/0000000000000000032 2024-12-01T18:50:01,789 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:50:01,791 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:01,805 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001791, exclude list is [], retry=0 2024-12-01T18:50:01,808 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:01,809 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:01,809 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:01,838 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001791 2024-12-01T18:50:01,839 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:40921:40921)] 2024-12-01T18:50:01,839 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee5181767944ecd8f2df89e7934875fa, NAME => 'testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:01,839 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:01,839 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,839 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,841 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,842 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5181767944ecd8f2df89e7934875fa columnFamilyName a 2024-12-01T18:50:01,842 DEBUG [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:01,847 DEBUG [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/a/d41a53f842e74652bcf1ed9edd6fee95 2024-12-01T18:50:01,847 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(327): Store=ee5181767944ecd8f2df89e7934875fa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:01,847 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,849 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5181767944ecd8f2df89e7934875fa columnFamilyName b 2024-12-01T18:50:01,849 DEBUG [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:01,849 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(327): Store=ee5181767944ecd8f2df89e7934875fa/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:01,849 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,850 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5181767944ecd8f2df89e7934875fa columnFamilyName c 2024-12-01T18:50:01,850 DEBUG [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:01,856 DEBUG [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/c/fa0f840cb88d45fca4d9fb1fe7e33df2 2024-12-01T18:50:01,856 INFO [StoreOpener-ee5181767944ecd8f2df89e7934875fa-1 {}] regionserver.HStore(327): Store=ee5181767944ecd8f2df89e7934875fa/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:01,856 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,857 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,858 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,859 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/0000000000000000032 2024-12-01T18:50:01,888 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:01,890 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/0000000000000000032 2024-12-01T18:50:01,890 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ee5181767944ecd8f2df89e7934875fa 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-01T18:50:01,905 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/.tmp/b/ce499ce23b414f818cb8f8cc380c661d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733079001537/Put/seqid=0 2024-12-01T18:50:01,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741905_1083 (size=5958) 2024-12-01T18:50:01,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741905_1083 (size=5958) 2024-12-01T18:50:01,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741905_1083 (size=5958) 2024-12-01T18:50:01,913 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/.tmp/b/ce499ce23b414f818cb8f8cc380c661d 2024-12-01T18:50:01,920 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/.tmp/b/ce499ce23b414f818cb8f8cc380c661d as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/b/ce499ce23b414f818cb8f8cc380c661d 2024-12-01T18:50:01,926 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/b/ce499ce23b414f818cb8f8cc380c661d, entries=10, sequenceid=32, filesize=5.8 K 2024-12-01T18:50:01,926 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ee5181767944ecd8f2df89e7934875fa in 36ms, sequenceid=32, compaction requested=false; wal=null 2024-12-01T18:50:01,927 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/0000000000000000032 2024-12-01T18:50:01,928 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,928 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,929 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:01,930 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee5181767944ecd8f2df89e7934875fa 2024-12-01T18:50:01,932 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/ee5181767944ecd8f2df89e7934875fa/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-01T18:50:01,933 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee5181767944ecd8f2df89e7934875fa; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69313517, jitterRate=0.03285188972949982}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:01,934 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee5181767944ecd8f2df89e7934875fa: Writing region info on filesystem at 1733079001839Initializing all the Stores at 1733079001840 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079001840Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079001841 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079001841Obtaining lock to block concurrent updates at 1733079001890 (+49 ms)Preparing flush snapshotting stores in ee5181767944ecd8f2df89e7934875fa at 1733079001890Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733079001890Flushing stores of testReplayEditsWrittenViaHRegion,,1733079001373.ee5181767944ecd8f2df89e7934875fa. at 1733079001890Flushing ee5181767944ecd8f2df89e7934875fa/b: creating writer at 1733079001890Flushing ee5181767944ecd8f2df89e7934875fa/b: appending metadata at 1733079001904 (+14 ms)Flushing ee5181767944ecd8f2df89e7934875fa/b: closing flushed file at 1733079001904Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36aa4448: reopening flushed file at 1733079001919 (+15 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ee5181767944ecd8f2df89e7934875fa in 36ms, sequenceid=32, compaction requested=false; wal=null at 1733079001926 (+7 ms)Cleaning up temporary data from old regions at 1733079001928 (+2 ms)Region opened successfully at 1733079001934 (+6 ms) 2024-12-01T18:50:01,959 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=419 (was 410) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:43456 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1540968591-172.17.0.2-1733078978096:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1540968591-172.17.0.2-1733078978096:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1540968591-172.17.0.2-1733078978096:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:40582 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:46730 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:52076 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:49352 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:49740 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=238 (was 197) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3743 (was 3757) 2024-12-01T18:50:01,959 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-01T18:50:01,975 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=238, ProcessCount=11, AvailableMemoryMB=3743 2024-12-01T18:50:01,975 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-01T18:50:01,997 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:50:01,999 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:50:01,999 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:50:02,002 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-56618220, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/hregion-56618220, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:02,016 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-56618220/hregion-56618220.1733079002002, exclude list is [], retry=0 2024-12-01T18:50:02,019 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:02,019 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:02,020 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:02,047 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-56618220/hregion-56618220.1733079002002 2024-12-01T18:50:02,048 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:50:02,048 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 02f379931ad7cb82bc159568a0e40188, NAME => 'testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38241/hbase 2024-12-01T18:50:02,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741907_1085 (size=68) 2024-12-01T18:50:02,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741907_1085 (size=68) 2024-12-01T18:50:02,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741907_1085 (size=68) 2024-12-01T18:50:02,059 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:02,060 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,062 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02f379931ad7cb82bc159568a0e40188 columnFamilyName a 2024-12-01T18:50:02,062 DEBUG [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:02,063 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(327): Store=02f379931ad7cb82bc159568a0e40188/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:02,063 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,064 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02f379931ad7cb82bc159568a0e40188 columnFamilyName b 2024-12-01T18:50:02,064 DEBUG [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:02,065 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(327): Store=02f379931ad7cb82bc159568a0e40188/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:02,065 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,066 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02f379931ad7cb82bc159568a0e40188 columnFamilyName c 2024-12-01T18:50:02,066 DEBUG [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:02,066 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(327): Store=02f379931ad7cb82bc159568a0e40188/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:02,067 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,067 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,067 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,068 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,068 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,069 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:02,070 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,072 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:50:02,072 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 02f379931ad7cb82bc159568a0e40188; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73401708, jitterRate=0.09377068281173706}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:02,073 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 02f379931ad7cb82bc159568a0e40188: Writing region info on filesystem at 1733079002059Initializing all the Stores at 1733079002060 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079002060Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079002060Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079002060Cleaning up temporary data from old regions at 1733079002068 (+8 ms)Region opened successfully at 1733079002072 (+4 ms) 2024-12-01T18:50:02,073 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 02f379931ad7cb82bc159568a0e40188, disabling compactions & flushes 2024-12-01T18:50:02,073 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. 2024-12-01T18:50:02,073 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. 2024-12-01T18:50:02,073 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. after waiting 0 ms 2024-12-01T18:50:02,073 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. 2024-12-01T18:50:02,073 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. 2024-12-01T18:50:02,073 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 02f379931ad7cb82bc159568a0e40188: Waiting for close lock at 1733079002073Disabling compacts and flushes for region at 1733079002073Disabling writes for close at 1733079002073Writing region close event to WAL at 1733079002073Closed at 1733079002073 2024-12-01T18:50:02,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741906_1084 (size=175) 2024-12-01T18:50:02,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741906_1084 (size=175) 2024-12-01T18:50:02,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741906_1084 (size=175) 2024-12-01T18:50:02,078 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:50:02,078 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-56618220:(num 1733079002002) 2024-12-01T18:50:02,078 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:50:02,079 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:02,093 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080, exclude list is [], retry=0 2024-12-01T18:50:02,095 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:02,096 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:02,096 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:02,124 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080 2024-12-01T18:50:02,124 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601)] 2024-12-01T18:50:02,177 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 02f379931ad7cb82bc159568a0e40188, NAME => 'testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:02,179 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,179 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:02,179 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,179 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,181 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,182 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02f379931ad7cb82bc159568a0e40188 columnFamilyName a 2024-12-01T18:50:02,182 DEBUG [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:02,183 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(327): Store=02f379931ad7cb82bc159568a0e40188/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:02,183 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,184 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02f379931ad7cb82bc159568a0e40188 columnFamilyName b 2024-12-01T18:50:02,184 DEBUG [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:02,184 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(327): Store=02f379931ad7cb82bc159568a0e40188/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:02,185 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,185 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02f379931ad7cb82bc159568a0e40188 columnFamilyName c 2024-12-01T18:50:02,185 DEBUG [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:02,186 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(327): Store=02f379931ad7cb82bc159568a0e40188/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:02,186 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,186 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,187 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,188 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,188 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,189 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:02,190 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,190 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 02f379931ad7cb82bc159568a0e40188; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60652077, jitterRate=-0.09621362388134003}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:02,190 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:02,191 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 02f379931ad7cb82bc159568a0e40188: Running coprocessor pre-open hook at 1733079002179Writing region info on filesystem at 1733079002179Initializing all the Stores at 1733079002180 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079002181 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079002181Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079002181Cleaning up temporary data from old regions at 1733079002188 (+7 ms)Running coprocessor post-open hooks at 1733079002190 (+2 ms)Region opened successfully at 1733079002191 (+1 ms) 2024-12-01T18:50:02,202 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 02f379931ad7cb82bc159568a0e40188 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-01T18:50:02,203 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 02f379931ad7cb82bc159568a0e40188/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:03,185 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T18:50:03,204 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 02f379931ad7cb82bc159568a0e40188/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:04,204 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 02f379931ad7cb82bc159568a0e40188/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:05,205 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 02f379931ad7cb82bc159568a0e40188/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:06,206 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 02f379931ad7cb82bc159568a0e40188/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:07,206 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 02f379931ad7cb82bc159568a0e40188/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:08,207 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 02f379931ad7cb82bc159568a0e40188/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:09,207 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 02f379931ad7cb82bc159568a0e40188/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:10,208 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 02f379931ad7cb82bc159568a0e40188/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:10,269 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-01T18:50:11,209 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 02f379931ad7cb82bc159568a0e40188/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:11,210 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 02f379931ad7cb82bc159568a0e40188: 2024-12-01T18:50:11,210 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. 2024-12-01T18:50:11,222 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 02f379931ad7cb82bc159568a0e40188: 2024-12-01T18:50:11,222 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-01T18:50:11,223 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 02f379931ad7cb82bc159568a0e40188, disabling compactions & flushes 2024-12-01T18:50:11,223 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. 2024-12-01T18:50:11,223 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. 2024-12-01T18:50:11,223 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. after waiting 0 ms 2024-12-01T18:50:11,223 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. 2024-12-01T18:50:11,223 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. 2024-12-01T18:50:11,223 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. 2024-12-01T18:50:11,223 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 02f379931ad7cb82bc159568a0e40188: Waiting for close lock at 1733079011222Running coprocessor pre-close hooks at 1733079011222Disabling compacts and flushes for region at 1733079011223 (+1 ms)Disabling writes for close at 1733079011223Writing region close event to WAL at 1733079011223Running coprocessor post-close hooks at 1733079011223Closed at 1733079011223 2024-12-01T18:50:11,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741908_1086 (size=4236) 2024-12-01T18:50:11,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741908_1086 (size=4236) 2024-12-01T18:50:11,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741908_1086 (size=4236) 2024-12-01T18:50:11,243 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080, size=4.1 K (4236bytes) 2024-12-01T18:50:11,243 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080 2024-12-01T18:50:11,243 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080 after 0ms 2024-12-01T18:50:11,272 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:11,272 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080 took 30ms 2024-12-01T18:50:11,275 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080 so closing down 2024-12-01T18:50:11,275 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:50:11,276 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733079002080.temp 2024-12-01T18:50:11,302 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/0000000000000000004-wal.1733079002080.temp 2024-12-01T18:50:11,302 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:50:11,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741909_1087 (size=3245) 2024-12-01T18:50:11,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741909_1087 (size=3245) 2024-12-01T18:50:11,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741909_1087 (size=3245) 2024-12-01T18:50:11,309 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/0000000000000000004-wal.1733079002080.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-01T18:50:11,311 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/0000000000000000004-wal.1733079002080.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/0000000000000000026 2024-12-01T18:50:11,311 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 38 ms; skipped=3; WAL=hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-01T18:50:11,311 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080, journal: Splitting hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080, size=4.1 K (4236bytes) at 1733079011243Finishing writing output for hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080 so closing down at 1733079011275 (+32 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/0000000000000000004-wal.1733079002080.temp at 1733079011302 (+27 ms)3 split writer threads finished at 1733079011302Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/0000000000000000004-wal.1733079002080.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733079011310 (+8 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/0000000000000000004-wal.1733079002080.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/0000000000000000026 at 1733079011311 (+1 ms)Processed 23 edits across 1 Regions in 38 ms; skipped=3; WAL=hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733079011311 2024-12-01T18:50:11,313 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079002080 to hdfs://localhost:38241/hbase/oldWALs/wal.1733079002080 2024-12-01T18:50:11,314 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/0000000000000000026 2024-12-01T18:50:11,314 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:50:11,315 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:11,329 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079011316, exclude list is [], retry=0 2024-12-01T18:50:11,332 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:11,332 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:11,332 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:11,359 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079011316 2024-12-01T18:50:11,360 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:50:11,360 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 02f379931ad7cb82bc159568a0e40188, NAME => 'testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:11,360 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,361 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:11,361 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,361 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,362 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,363 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02f379931ad7cb82bc159568a0e40188 columnFamilyName a 2024-12-01T18:50:11,363 DEBUG [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:11,364 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(327): Store=02f379931ad7cb82bc159568a0e40188/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:11,364 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,365 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02f379931ad7cb82bc159568a0e40188 columnFamilyName b 2024-12-01T18:50:11,365 DEBUG [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:11,365 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(327): Store=02f379931ad7cb82bc159568a0e40188/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:11,366 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,366 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02f379931ad7cb82bc159568a0e40188 columnFamilyName c 2024-12-01T18:50:11,366 DEBUG [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:11,367 INFO [StoreOpener-02f379931ad7cb82bc159568a0e40188-1 {}] regionserver.HStore(327): Store=02f379931ad7cb82bc159568a0e40188/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:11,367 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,367 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,369 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,369 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/0000000000000000026 2024-12-01T18:50:11,397 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:11,398 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/0000000000000000026 2024-12-01T18:50:11,399 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 02f379931ad7cb82bc159568a0e40188 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-01T18:50:11,414 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/.tmp/a/2be323b3e70c42f6bfce29540a60ec7f is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733079011214/Put/seqid=0 2024-12-01T18:50:11,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741911_1089 (size=5523) 2024-12-01T18:50:11,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741911_1089 (size=5523) 2024-12-01T18:50:11,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741911_1089 (size=5523) 2024-12-01T18:50:11,424 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/.tmp/a/2be323b3e70c42f6bfce29540a60ec7f 2024-12-01T18:50:11,444 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/.tmp/b/f13642c90c4e496ea3e4f1c1bb1fa603 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733079011210/Put/seqid=0 2024-12-01T18:50:11,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741912_1090 (size=5524) 2024-12-01T18:50:11,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741912_1090 (size=5524) 2024-12-01T18:50:11,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741912_1090 (size=5524) 2024-12-01T18:50:11,452 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/.tmp/b/f13642c90c4e496ea3e4f1c1bb1fa603 2024-12-01T18:50:11,472 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/.tmp/c/2cb8a4cae6324dbeb0b0862529c0c222 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733079011212/Put/seqid=0 2024-12-01T18:50:11,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741913_1091 (size=5457) 2024-12-01T18:50:11,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741913_1091 (size=5457) 2024-12-01T18:50:11,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741913_1091 (size=5457) 2024-12-01T18:50:11,479 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/.tmp/c/2cb8a4cae6324dbeb0b0862529c0c222 2024-12-01T18:50:11,485 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/.tmp/a/2be323b3e70c42f6bfce29540a60ec7f as hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/a/2be323b3e70c42f6bfce29540a60ec7f 2024-12-01T18:50:11,490 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/a/2be323b3e70c42f6bfce29540a60ec7f, entries=7, sequenceid=26, filesize=5.4 K 2024-12-01T18:50:11,491 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/.tmp/b/f13642c90c4e496ea3e4f1c1bb1fa603 as hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/b/f13642c90c4e496ea3e4f1c1bb1fa603 2024-12-01T18:50:11,496 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/b/f13642c90c4e496ea3e4f1c1bb1fa603, entries=7, sequenceid=26, filesize=5.4 K 2024-12-01T18:50:11,497 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/.tmp/c/2cb8a4cae6324dbeb0b0862529c0c222 as hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/c/2cb8a4cae6324dbeb0b0862529c0c222 2024-12-01T18:50:11,501 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/c/2cb8a4cae6324dbeb0b0862529c0c222, entries=6, sequenceid=26, filesize=5.3 K 2024-12-01T18:50:11,501 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 02f379931ad7cb82bc159568a0e40188 in 102ms, sequenceid=26, compaction requested=false; wal=null 2024-12-01T18:50:11,502 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/0000000000000000026 2024-12-01T18:50:11,503 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,503 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,504 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:11,505 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,507 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsAfterAbortingFlush/02f379931ad7cb82bc159568a0e40188/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-01T18:50:11,508 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 02f379931ad7cb82bc159568a0e40188; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63179105, jitterRate=-0.05855797231197357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:11,508 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 02f379931ad7cb82bc159568a0e40188 2024-12-01T18:50:11,509 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 02f379931ad7cb82bc159568a0e40188: Running coprocessor pre-open hook at 1733079011361Writing region info on filesystem at 1733079011361Initializing all the Stores at 1733079011362 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079011362Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079011362Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079011362Obtaining lock to block concurrent updates at 1733079011399 (+37 ms)Preparing flush snapshotting stores in 02f379931ad7cb82bc159568a0e40188 at 1733079011399Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733079011399Flushing stores of testReplayEditsAfterAbortingFlush,,1733079001998.02f379931ad7cb82bc159568a0e40188. at 1733079011399Flushing 02f379931ad7cb82bc159568a0e40188/a: creating writer at 1733079011399Flushing 02f379931ad7cb82bc159568a0e40188/a: appending metadata at 1733079011413 (+14 ms)Flushing 02f379931ad7cb82bc159568a0e40188/a: closing flushed file at 1733079011413Flushing 02f379931ad7cb82bc159568a0e40188/b: creating writer at 1733079011429 (+16 ms)Flushing 02f379931ad7cb82bc159568a0e40188/b: appending metadata at 1733079011444 (+15 ms)Flushing 02f379931ad7cb82bc159568a0e40188/b: closing flushed file at 1733079011444Flushing 02f379931ad7cb82bc159568a0e40188/c: creating writer at 1733079011457 (+13 ms)Flushing 02f379931ad7cb82bc159568a0e40188/c: appending metadata at 1733079011472 (+15 ms)Flushing 02f379931ad7cb82bc159568a0e40188/c: closing flushed file at 1733079011472Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29988ac6: reopening flushed file at 1733079011484 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31d017f1: reopening flushed file at 1733079011490 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c021f5a: reopening flushed file at 1733079011496 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 02f379931ad7cb82bc159568a0e40188 in 102ms, sequenceid=26, compaction requested=false; wal=null at 1733079011501 (+5 ms)Cleaning up temporary data from old regions at 1733079011503 (+2 ms)Running coprocessor post-open hooks at 1733079011508 (+5 ms)Region opened successfully at 1733079011508 2024-12-01T18:50:11,521 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-01T18:50:11,521 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-01T18:50:11,522 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-01T18:50:11,522 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-01T18:50:11,529 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419 (was 419), OpenFileDescriptor=1182 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=219 (was 238), ProcessCount=11 (was 11), AvailableMemoryMB=3751 (was 3743) - AvailableMemoryMB LEAK? - 2024-12-01T18:50:11,529 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-01T18:50:11,541 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=419, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=219, ProcessCount=11, AvailableMemoryMB=3750 2024-12-01T18:50:11,541 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-01T18:50:11,556 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:50:11,557 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:50:11,558 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:50:11,560 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-99039290, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/hregion-99039290, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:11,573 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-99039290/hregion-99039290.1733079011560, exclude list is [], retry=0 2024-12-01T18:50:11,576 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:11,576 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:11,576 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:11,603 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-99039290/hregion-99039290.1733079011560 2024-12-01T18:50:11,603 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:50:11,604 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 50405f5261bcf8cf9398881ea13ea9c4, NAME => 'testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38241/hbase 2024-12-01T18:50:11,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741915_1093 (size=61) 2024-12-01T18:50:11,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741915_1093 (size=61) 2024-12-01T18:50:11,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741915_1093 (size=61) 2024-12-01T18:50:11,613 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:11,614 INFO [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,615 INFO [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50405f5261bcf8cf9398881ea13ea9c4 columnFamilyName a 2024-12-01T18:50:11,615 DEBUG [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:11,616 INFO [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] regionserver.HStore(327): Store=50405f5261bcf8cf9398881ea13ea9c4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:11,616 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,617 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,617 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,617 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,617 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,619 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,621 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:50:11,621 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 50405f5261bcf8cf9398881ea13ea9c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63496756, jitterRate=-0.05382460355758667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T18:50:11,622 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 50405f5261bcf8cf9398881ea13ea9c4: Writing region info on filesystem at 1733079011613Initializing all the Stores at 1733079011614 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079011614Cleaning up temporary data from old regions at 1733079011617 (+3 ms)Region opened successfully at 1733079011621 (+4 ms) 2024-12-01T18:50:11,622 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 50405f5261bcf8cf9398881ea13ea9c4, disabling compactions & flushes 2024-12-01T18:50:11,622 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4. 2024-12-01T18:50:11,622 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4. 2024-12-01T18:50:11,622 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4. after waiting 0 ms 2024-12-01T18:50:11,622 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4. 2024-12-01T18:50:11,622 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4. 2024-12-01T18:50:11,622 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 50405f5261bcf8cf9398881ea13ea9c4: Waiting for close lock at 1733079011622Disabling compacts and flushes for region at 1733079011622Disabling writes for close at 1733079011622Writing region close event to WAL at 1733079011622Closed at 1733079011622 2024-12-01T18:50:11,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741914_1092 (size=175) 2024-12-01T18:50:11,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741914_1092 (size=175) 2024-12-01T18:50:11,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741914_1092 (size=175) 2024-12-01T18:50:11,627 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:50:11,627 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-99039290:(num 1733079011560) 2024-12-01T18:50:11,627 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:50:11,628 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:11,642 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629, exclude list is [], retry=0 2024-12-01T18:50:11,655 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:11,656 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:11,656 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:11,684 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629 2024-12-01T18:50:11,684 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601)] 2024-12-01T18:50:11,684 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 50405f5261bcf8cf9398881ea13ea9c4, NAME => 'testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:11,684 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:11,684 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,684 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,686 INFO [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,687 INFO [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50405f5261bcf8cf9398881ea13ea9c4 columnFamilyName a 2024-12-01T18:50:11,687 DEBUG [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:11,687 INFO [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] regionserver.HStore(327): Store=50405f5261bcf8cf9398881ea13ea9c4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:11,687 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,688 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,689 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,689 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,689 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,691 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:11,692 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 50405f5261bcf8cf9398881ea13ea9c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70193974, jitterRate=0.04597172141075134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T18:50:11,692 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 50405f5261bcf8cf9398881ea13ea9c4: Writing region info on filesystem at 1733079011684Initializing all the Stores at 1733079011685 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079011685Cleaning up temporary data from old regions at 1733079011689 (+4 ms)Region opened successfully at 1733079011692 (+3 ms) 2024-12-01T18:50:11,700 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 50405f5261bcf8cf9398881ea13ea9c4, disabling compactions & flushes 2024-12-01T18:50:11,700 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4. 2024-12-01T18:50:11,700 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4. 2024-12-01T18:50:11,700 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4. after waiting 0 ms 2024-12-01T18:50:11,700 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4. 2024-12-01T18:50:11,700 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4. 2024-12-01T18:50:11,700 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4. 2024-12-01T18:50:11,700 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 50405f5261bcf8cf9398881ea13ea9c4: Waiting for close lock at 1733079011700Disabling compacts and flushes for region at 1733079011700Disabling writes for close at 1733079011700Writing region close event to WAL at 1733079011700Closed at 1733079011700 2024-12-01T18:50:11,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741916_1094 (size=1795) 2024-12-01T18:50:11,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741916_1094 (size=1795) 2024-12-01T18:50:11,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741916_1094 (size=1795) 2024-12-01T18:50:11,719 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629, size=1.8 K (1795bytes) 2024-12-01T18:50:11,719 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629 2024-12-01T18:50:11,719 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629 after 0ms 2024-12-01T18:50:11,746 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:11,746 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629 took 28ms 2024-12-01T18:50:11,748 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629 so closing down 2024-12-01T18:50:11,748 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:50:11,750 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733079011629.temp 2024-12-01T18:50:11,777 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/0000000000000000003-wal.1733079011629.temp 2024-12-01T18:50:11,777 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:50:11,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741917_1095 (size=1795) 2024-12-01T18:50:11,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741917_1095 (size=1795) 2024-12-01T18:50:11,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741917_1095 (size=1795) 2024-12-01T18:50:11,784 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/0000000000000000003-wal.1733079011629.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-01T18:50:11,786 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/0000000000000000003-wal.1733079011629.temp to hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/0000000000000000012 2024-12-01T18:50:11,786 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-01T18:50:11,786 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629, journal: Splitting hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629, size=1.8 K (1795bytes) at 1733079011719Finishing writing output for hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629 so closing down at 1733079011748 (+29 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/0000000000000000003-wal.1733079011629.temp at 1733079011777 (+29 ms)3 split writer threads finished at 1733079011777Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/0000000000000000003-wal.1733079011629.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733079011784 (+7 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/0000000000000000003-wal.1733079011629.temp to hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/0000000000000000012 at 1733079011786 (+2 ms)Processed 10 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733079011786 2024-12-01T18:50:11,788 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079011629 to hdfs://localhost:38241/hbase/oldWALs/wal.1733079011629 2024-12-01T18:50:11,788 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/0000000000000000012 2024-12-01T18:50:11,818 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:12,116 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:50:12,118 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:12,131 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079012119, exclude list is [], retry=0 2024-12-01T18:50:12,134 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:12,134 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:12,135 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:12,161 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079012119 2024-12-01T18:50:12,162 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601)] 2024-12-01T18:50:12,162 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 50405f5261bcf8cf9398881ea13ea9c4, NAME => 'testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:12,162 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:12,162 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,162 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,165 INFO [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,166 INFO [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50405f5261bcf8cf9398881ea13ea9c4 columnFamilyName a 2024-12-01T18:50:12,166 DEBUG [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:12,167 INFO [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] regionserver.HStore(327): Store=50405f5261bcf8cf9398881ea13ea9c4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:12,168 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,168 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,170 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,171 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/0000000000000000012 2024-12-01T18:50:12,205 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:12,206 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/0000000000000000012 2024-12-01T18:50:12,206 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 50405f5261bcf8cf9398881ea13ea9c4 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-01T18:50:12,224 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/.tmp/a/d144def13d73438da3a6b35ec825a77d is 79, key is testDatalossWhenInputError/a:x0/1733079011692/Put/seqid=0 2024-12-01T18:50:12,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741919_1097 (size=5808) 2024-12-01T18:50:12,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741919_1097 (size=5808) 2024-12-01T18:50:12,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741919_1097 (size=5808) 2024-12-01T18:50:12,231 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/.tmp/a/d144def13d73438da3a6b35ec825a77d 2024-12-01T18:50:12,241 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/.tmp/a/d144def13d73438da3a6b35ec825a77d as hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/a/d144def13d73438da3a6b35ec825a77d 2024-12-01T18:50:12,248 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/a/d144def13d73438da3a6b35ec825a77d, entries=10, sequenceid=12, filesize=5.7 K 2024-12-01T18:50:12,248 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 50405f5261bcf8cf9398881ea13ea9c4 in 42ms, sequenceid=12, compaction requested=false; wal=null 2024-12-01T18:50:12,249 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/0000000000000000012 2024-12-01T18:50:12,250 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,250 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,252 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,254 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-01T18:50:12,255 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 50405f5261bcf8cf9398881ea13ea9c4; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74871611, jitterRate=0.11567394435405731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T18:50:12,256 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 50405f5261bcf8cf9398881ea13ea9c4: Writing region info on filesystem at 1733079012162Initializing all the Stores at 1733079012164 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079012165 (+1 ms)Obtaining lock to block concurrent updates at 1733079012206 (+41 ms)Preparing flush snapshotting stores in 50405f5261bcf8cf9398881ea13ea9c4 at 1733079012206Finished memstore snapshotting testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733079012206Flushing stores of testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4. at 1733079012206Flushing 50405f5261bcf8cf9398881ea13ea9c4/a: creating writer at 1733079012206Flushing 50405f5261bcf8cf9398881ea13ea9c4/a: appending metadata at 1733079012223 (+17 ms)Flushing 50405f5261bcf8cf9398881ea13ea9c4/a: closing flushed file at 1733079012223Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21861b87: reopening flushed file at 1733079012239 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 50405f5261bcf8cf9398881ea13ea9c4 in 42ms, sequenceid=12, compaction requested=false; wal=null at 1733079012248 (+9 ms)Cleaning up temporary data from old regions at 1733079012250 (+2 ms)Region opened successfully at 1733079012255 (+5 ms) 2024-12-01T18:50:12,258 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 50405f5261bcf8cf9398881ea13ea9c4, NAME => 'testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:12,258 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733079011556.50405f5261bcf8cf9398881ea13ea9c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:12,258 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,258 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,259 INFO [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,260 INFO [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50405f5261bcf8cf9398881ea13ea9c4 columnFamilyName a 2024-12-01T18:50:12,260 DEBUG [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:12,265 DEBUG [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/a/d144def13d73438da3a6b35ec825a77d 2024-12-01T18:50:12,265 INFO [StoreOpener-50405f5261bcf8cf9398881ea13ea9c4-1 {}] regionserver.HStore(327): Store=50405f5261bcf8cf9398881ea13ea9c4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:12,265 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,266 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,267 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,267 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,267 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,269 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 50405f5261bcf8cf9398881ea13ea9c4 2024-12-01T18:50:12,271 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testDatalossWhenInputError/50405f5261bcf8cf9398881ea13ea9c4/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-01T18:50:12,271 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 50405f5261bcf8cf9398881ea13ea9c4; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62862594, jitterRate=-0.06327435374259949}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T18:50:12,271 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 50405f5261bcf8cf9398881ea13ea9c4: Writing region info on filesystem at 1733079012258Initializing all the Stores at 1733079012259 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079012259Cleaning up temporary data from old regions at 1733079012267 (+8 ms)Region opened successfully at 1733079012271 (+4 ms) 2024-12-01T18:50:12,288 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=429 (was 419) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:37292 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1540968591-172.17.0.2-1733078978096:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:37406 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:58120 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1540968591-172.17.0.2-1733078978096:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:58054 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1540968591-172.17.0.2-1733078978096:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:35312 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:35246 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=209 (was 219), ProcessCount=11 (was 11), AvailableMemoryMB=3736 (was 3750) 2024-12-01T18:50:12,288 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-01T18:50:12,300 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=429, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=209, ProcessCount=11, AvailableMemoryMB=3735 2024-12-01T18:50:12,300 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-01T18:50:12,315 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:50:12,316 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:50:12,317 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:50:12,319 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73874459, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/hregion-73874459, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:12,332 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73874459/hregion-73874459.1733079012319, exclude list is [], retry=0 2024-12-01T18:50:12,334 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:12,334 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:12,335 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:12,372 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73874459/hregion-73874459.1733079012319 2024-12-01T18:50:12,372 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921)] 2024-12-01T18:50:12,372 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6270053f257a61a520e972a1c2aeeef6, NAME => 'testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38241/hbase 2024-12-01T18:50:12,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741921_1099 (size=63) 2024-12-01T18:50:12,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741921_1099 (size=63) 2024-12-01T18:50:12,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741921_1099 (size=63) 2024-12-01T18:50:12,382 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:12,383 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,385 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6270053f257a61a520e972a1c2aeeef6 columnFamilyName a 2024-12-01T18:50:12,385 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:12,385 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(327): Store=6270053f257a61a520e972a1c2aeeef6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:12,385 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,386 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6270053f257a61a520e972a1c2aeeef6 columnFamilyName b 2024-12-01T18:50:12,386 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:12,387 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(327): Store=6270053f257a61a520e972a1c2aeeef6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:12,387 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,388 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6270053f257a61a520e972a1c2aeeef6 columnFamilyName c 2024-12-01T18:50:12,388 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:12,388 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(327): Store=6270053f257a61a520e972a1c2aeeef6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:12,389 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,389 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,390 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,391 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,391 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,391 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:12,392 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,394 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:50:12,395 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6270053f257a61a520e972a1c2aeeef6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67513878, jitterRate=0.006035178899765015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:12,395 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6270053f257a61a520e972a1c2aeeef6: Writing region info on filesystem at 1733079012382Initializing all the Stores at 1733079012383 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079012383Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079012383Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079012383Cleaning up temporary data from old regions at 1733079012391 (+8 ms)Region opened successfully at 1733079012395 (+4 ms) 2024-12-01T18:50:12,396 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6270053f257a61a520e972a1c2aeeef6, disabling compactions & flushes 2024-12-01T18:50:12,396 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6. 2024-12-01T18:50:12,396 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6. 2024-12-01T18:50:12,396 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6. after waiting 0 ms 2024-12-01T18:50:12,396 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6. 2024-12-01T18:50:12,396 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6. 2024-12-01T18:50:12,396 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6270053f257a61a520e972a1c2aeeef6: Waiting for close lock at 1733079012396Disabling compacts and flushes for region at 1733079012396Disabling writes for close at 1733079012396Writing region close event to WAL at 1733079012396Closed at 1733079012396 2024-12-01T18:50:12,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741920_1098 (size=175) 2024-12-01T18:50:12,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741920_1098 (size=175) 2024-12-01T18:50:12,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741920_1098 (size=175) 2024-12-01T18:50:12,400 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:50:12,400 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73874459:(num 1733079012319) 2024-12-01T18:50:12,400 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:50:12,402 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:12,415 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402, exclude list is [], retry=0 2024-12-01T18:50:12,417 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:12,417 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:12,417 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:12,448 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402 2024-12-01T18:50:12,448 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921)] 2024-12-01T18:50:12,448 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6270053f257a61a520e972a1c2aeeef6, NAME => 'testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:12,448 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:12,449 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,449 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,450 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,451 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6270053f257a61a520e972a1c2aeeef6 columnFamilyName a 2024-12-01T18:50:12,451 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:12,451 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(327): Store=6270053f257a61a520e972a1c2aeeef6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:12,451 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,452 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6270053f257a61a520e972a1c2aeeef6 columnFamilyName b 2024-12-01T18:50:12,452 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:12,452 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(327): Store=6270053f257a61a520e972a1c2aeeef6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:12,453 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,453 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6270053f257a61a520e972a1c2aeeef6 columnFamilyName c 2024-12-01T18:50:12,453 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:12,454 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(327): Store=6270053f257a61a520e972a1c2aeeef6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:12,454 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,455 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,455 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,456 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,456 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,457 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:12,458 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:12,459 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6270053f257a61a520e972a1c2aeeef6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65220105, jitterRate=-0.028144702315330505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:12,459 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6270053f257a61a520e972a1c2aeeef6: Writing region info on filesystem at 1733079012449Initializing all the Stores at 1733079012449Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079012450 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079012450Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079012450Cleaning up temporary data from old regions at 1733079012456 (+6 ms)Region opened successfully at 1733079012459 (+3 ms) 2024-12-01T18:50:12,463 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733079012462/Put/seqid=0 2024-12-01T18:50:12,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741923_1101 (size=4875) 2024-12-01T18:50:12,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741923_1101 (size=4875) 2024-12-01T18:50:12,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741923_1101 (size=4875) 2024-12-01T18:50:12,474 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733079012473/Put/seqid=0 2024-12-01T18:50:12,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741924_1102 (size=4875) 2024-12-01T18:50:12,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741924_1102 (size=4875) 2024-12-01T18:50:12,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741924_1102 (size=4875) 2024-12-01T18:50:12,484 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733079012484/Put/seqid=0 2024-12-01T18:50:12,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741925_1103 (size=4875) 2024-12-01T18:50:12,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741925_1103 (size=4875) 2024-12-01T18:50:12,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741925_1103 (size=4875) 2024-12-01T18:50:12,495 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 6270053f257a61a520e972a1c2aeeef6/a 2024-12-01T18:50:12,499 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-01T18:50:12,499 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-01T18:50:12,499 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 6270053f257a61a520e972a1c2aeeef6/a 2024-12-01T18:50:12,502 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-01T18:50:12,502 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-01T18:50:12,503 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 6270053f257a61a520e972a1c2aeeef6/a 2024-12-01T18:50:12,505 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-01T18:50:12,505 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-01T18:50:12,506 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6270053f257a61a520e972a1c2aeeef6 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-01T18:50:12,522 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/.tmp/a/403f3028e09947e38eb5a725bcac8a76 is 55, key is testCompactedBulkLoadedFiles/a:a/1733079012460/Put/seqid=0 2024-12-01T18:50:12,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741926_1104 (size=5107) 2024-12-01T18:50:12,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741926_1104 (size=5107) 2024-12-01T18:50:12,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741926_1104 (size=5107) 2024-12-01T18:50:12,529 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/.tmp/a/403f3028e09947e38eb5a725bcac8a76 2024-12-01T18:50:12,534 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/.tmp/a/403f3028e09947e38eb5a725bcac8a76 as hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/403f3028e09947e38eb5a725bcac8a76 2024-12-01T18:50:12,539 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/403f3028e09947e38eb5a725bcac8a76, entries=1, sequenceid=4, filesize=5.0 K 2024-12-01T18:50:12,541 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 6270053f257a61a520e972a1c2aeeef6 in 35ms, sequenceid=4, compaction requested=false 2024-12-01T18:50:12,541 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6270053f257a61a520e972a1c2aeeef6: 2024-12-01T18:50:12,542 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_ 2024-12-01T18:50:12,543 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/55e1342989334281ba556aaf1377e7ab_SeqId_4_ 2024-12-01T18:50:12,543 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/456251a0beb74cb6aa1870476f88722c_SeqId_4_ 2024-12-01T18:50:12,544 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile0 into 6270053f257a61a520e972a1c2aeeef6/a as hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_ - updating store file list. 2024-12-01T18:50:12,548 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T18:50:12,548 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_ into 6270053f257a61a520e972a1c2aeeef6/a 2024-12-01T18:50:12,548 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile0 into 6270053f257a61a520e972a1c2aeeef6/a (new location: hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_) 2024-12-01T18:50:12,549 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile1 into 6270053f257a61a520e972a1c2aeeef6/a as hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/55e1342989334281ba556aaf1377e7ab_SeqId_4_ - updating store file list. 2024-12-01T18:50:12,552 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 55e1342989334281ba556aaf1377e7ab_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T18:50:12,553 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/55e1342989334281ba556aaf1377e7ab_SeqId_4_ into 6270053f257a61a520e972a1c2aeeef6/a 2024-12-01T18:50:12,553 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile1 into 6270053f257a61a520e972a1c2aeeef6/a (new location: hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/55e1342989334281ba556aaf1377e7ab_SeqId_4_) 2024-12-01T18:50:12,553 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile2 into 6270053f257a61a520e972a1c2aeeef6/a as hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/456251a0beb74cb6aa1870476f88722c_SeqId_4_ - updating store file list. 2024-12-01T18:50:12,558 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 456251a0beb74cb6aa1870476f88722c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T18:50:12,558 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/456251a0beb74cb6aa1870476f88722c_SeqId_4_ into 6270053f257a61a520e972a1c2aeeef6/a 2024-12-01T18:50:12,558 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38241/hbase/testCompactedBulkLoadedFiles/hfile2 into 6270053f257a61a520e972a1c2aeeef6/a (new location: hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/456251a0beb74cb6aa1870476f88722c_SeqId_4_) 2024-12-01T18:50:12,564 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-01T18:50:12,564 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6270053f257a61a520e972a1c2aeeef6/a is initiating major compaction (all files) 2024-12-01T18:50:12,564 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6270053f257a61a520e972a1c2aeeef6/a in testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6. 2024-12-01T18:50:12,565 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/403f3028e09947e38eb5a725bcac8a76, hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_, hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/55e1342989334281ba556aaf1377e7ab_SeqId_4_, hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/456251a0beb74cb6aa1870476f88722c_SeqId_4_] into tmpdir=hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/.tmp, totalSize=19.3 K 2024-12-01T18:50:12,565 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 403f3028e09947e38eb5a725bcac8a76, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733079012460 2024-12-01T18:50:12,565 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-01T18:50:12,566 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 55e1342989334281ba556aaf1377e7ab_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-01T18:50:12,566 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 456251a0beb74cb6aa1870476f88722c_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-01T18:50:12,577 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/.tmp/a/f2449a7ebc664885afcfbccfb13f74b9 is 55, key is testCompactedBulkLoadedFiles/a:a/1733079012460/Put/seqid=0 2024-12-01T18:50:12,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741927_1105 (size=6154) 2024-12-01T18:50:12,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741927_1105 (size=6154) 2024-12-01T18:50:12,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741927_1105 (size=6154) 2024-12-01T18:50:12,592 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/.tmp/a/f2449a7ebc664885afcfbccfb13f74b9 as hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/f2449a7ebc664885afcfbccfb13f74b9 2024-12-01T18:50:12,598 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 6270053f257a61a520e972a1c2aeeef6/a of 6270053f257a61a520e972a1c2aeeef6 into f2449a7ebc664885afcfbccfb13f74b9(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-01T18:50:12,598 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6270053f257a61a520e972a1c2aeeef6: 2024-12-01T18:50:12,598 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-01T18:50:12,598 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-01T18:50:12,633 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402, size=0 (0bytes) 2024-12-01T18:50:12,633 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402 might be still open, length is 0 2024-12-01T18:50:12,633 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402 2024-12-01T18:50:12,633 WARN [IPC Server handler 2 on default port 38241 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-01T18:50:12,634 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402 after 1ms 2024-12-01T18:50:13,061 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:35364 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:35899:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35364 dst: /127.0.0.1:35899 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35899 remote=/127.0.0.1:35364]. Total timeout mills is 60000, 59537 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:13,061 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:58156 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41037:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58156 dst: /127.0.0.1:41037 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:13,061 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:37442 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37442 dst: /127.0.0.1:43787 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:13,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741922_1106 (size=1794) 2024-12-01T18:50:13,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741922_1106 (size=1794) 2024-12-01T18:50:16,634 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402 after 4001ms 2024-12-01T18:50:16,667 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:16,667 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402 took 4034ms 2024-12-01T18:50:16,669 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402; continuing. 2024-12-01T18:50:16,669 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402 so closing down 2024-12-01T18:50:16,669 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:50:16,670 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733079012402.temp 2024-12-01T18:50:16,700 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/0000000000000000003-wal.1733079012402.temp 2024-12-01T18:50:16,701 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:50:16,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741928_1107 (size=800) 2024-12-01T18:50:16,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741928_1107 (size=800) 2024-12-01T18:50:16,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741928_1107 (size=800) 2024-12-01T18:50:16,708 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/0000000000000000003-wal.1733079012402.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-01T18:50:16,709 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/0000000000000000003-wal.1733079012402.temp to hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/0000000000000000008 2024-12-01T18:50:16,709 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 41 ms; skipped=3; WAL=hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402, size=0, length=0, corrupted=false, cancelled=false 2024-12-01T18:50:16,709 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402, journal: Splitting hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402, size=0 (0bytes) at 1733079012633Finishing writing output for hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402 so closing down at 1733079016669 (+4036 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/0000000000000000003-wal.1733079012402.temp at 1733079016700 (+31 ms)3 split writer threads finished at 1733079016701 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/0000000000000000003-wal.1733079012402.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733079016708 (+7 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/0000000000000000003-wal.1733079012402.temp to hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/0000000000000000008 at 1733079016709 (+1 ms)Processed 5 edits across 1 Regions in 41 ms; skipped=3; WAL=hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402, size=0, length=0, corrupted=false, cancelled=false at 1733079016709 2024-12-01T18:50:16,711 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402 to hdfs://localhost:38241/hbase/oldWALs/wal.1733079012402 2024-12-01T18:50:16,712 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/0000000000000000008 2024-12-01T18:50:16,712 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:50:16,714 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:16,728 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079016715, exclude list is [], retry=0 2024-12-01T18:50:16,731 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:16,731 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:16,732 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:16,744 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T18:50:16,765 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079016715 2024-12-01T18:50:16,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:40921:40921)] 2024-12-01T18:50:16,765 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6270053f257a61a520e972a1c2aeeef6, NAME => 'testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:16,765 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:16,766 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:16,766 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:16,767 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:16,768 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6270053f257a61a520e972a1c2aeeef6 columnFamilyName a 2024-12-01T18:50:16,768 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:16,775 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/403f3028e09947e38eb5a725bcac8a76 2024-12-01T18:50:16,779 DEBUG [StoreFileOpener-6270053f257a61a520e972a1c2aeeef6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 456251a0beb74cb6aa1870476f88722c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T18:50:16,779 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/456251a0beb74cb6aa1870476f88722c_SeqId_4_ 2024-12-01T18:50:16,786 DEBUG [StoreFileOpener-6270053f257a61a520e972a1c2aeeef6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 55e1342989334281ba556aaf1377e7ab_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T18:50:16,786 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/55e1342989334281ba556aaf1377e7ab_SeqId_4_ 2024-12-01T18:50:16,789 DEBUG [StoreFileOpener-6270053f257a61a520e972a1c2aeeef6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T18:50:16,789 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_ 2024-12-01T18:50:16,792 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/f2449a7ebc664885afcfbccfb13f74b9 2024-12-01T18:50:16,793 WARN [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/403f3028e09947e38eb5a725bcac8a76 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@356d2f34 2024-12-01T18:50:16,793 WARN [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/456251a0beb74cb6aa1870476f88722c_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@356d2f34 2024-12-01T18:50:16,793 WARN [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/55e1342989334281ba556aaf1377e7ab_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@356d2f34 2024-12-01T18:50:16,793 WARN [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@356d2f34 2024-12-01T18:50:16,793 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/403f3028e09947e38eb5a725bcac8a76, hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/456251a0beb74cb6aa1870476f88722c_SeqId_4_, hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/55e1342989334281ba556aaf1377e7ab_SeqId_4_, hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_] to archive 2024-12-01T18:50:16,794 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-01T18:50:16,795 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/403f3028e09947e38eb5a725bcac8a76 to hdfs://localhost:38241/hbase/archive/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/403f3028e09947e38eb5a725bcac8a76 2024-12-01T18:50:16,796 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/456251a0beb74cb6aa1870476f88722c_SeqId_4_ to hdfs://localhost:38241/hbase/archive/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/456251a0beb74cb6aa1870476f88722c_SeqId_4_ 2024-12-01T18:50:16,797 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/55e1342989334281ba556aaf1377e7ab_SeqId_4_ to hdfs://localhost:38241/hbase/archive/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/55e1342989334281ba556aaf1377e7ab_SeqId_4_ 2024-12-01T18:50:16,798 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_ to hdfs://localhost:38241/hbase/archive/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/a/b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_ 2024-12-01T18:50:16,799 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(327): Store=6270053f257a61a520e972a1c2aeeef6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:16,799 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:16,800 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6270053f257a61a520e972a1c2aeeef6 columnFamilyName b 2024-12-01T18:50:16,800 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:16,800 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(327): Store=6270053f257a61a520e972a1c2aeeef6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:16,800 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:16,801 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6270053f257a61a520e972a1c2aeeef6 columnFamilyName c 2024-12-01T18:50:16,801 DEBUG [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:16,802 INFO [StoreOpener-6270053f257a61a520e972a1c2aeeef6-1 {}] regionserver.HStore(327): Store=6270053f257a61a520e972a1c2aeeef6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:16,802 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:16,802 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:16,803 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:16,804 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/0000000000000000008 2024-12-01T18:50:16,831 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:16,833 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 6270053f257a61a520e972a1c2aeeef6 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "6270053f257a61a520e972a1c2aeeef6" family_name: "a" compaction_input: "403f3028e09947e38eb5a725bcac8a76" compaction_input: "b0893adec5244b1ea3d7947ed4c49ad7_SeqId_4_" compaction_input: "55e1342989334281ba556aaf1377e7ab_SeqId_4_" compaction_input: "456251a0beb74cb6aa1870476f88722c_SeqId_4_" compaction_output: "f2449a7ebc664885afcfbccfb13f74b9" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-01T18:50:16,833 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-01T18:50:16,833 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/0000000000000000008 2024-12-01T18:50:16,834 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/0000000000000000008 2024-12-01T18:50:16,835 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:16,836 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:16,836 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:16,837 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6270053f257a61a520e972a1c2aeeef6 2024-12-01T18:50:16,840 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testCompactedBulkLoadedFiles/6270053f257a61a520e972a1c2aeeef6/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-01T18:50:16,840 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6270053f257a61a520e972a1c2aeeef6; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74306211, jitterRate=0.10724882781505585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:16,841 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6270053f257a61a520e972a1c2aeeef6: Writing region info on filesystem at 1733079016766Initializing all the Stores at 1733079016767 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079016767Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079016767Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079016767Cleaning up temporary data from old regions at 1733079016836 (+69 ms)Region opened successfully at 1733079016841 (+5 ms) 2024-12-01T18:50:16,843 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6270053f257a61a520e972a1c2aeeef6, disabling compactions & flushes 2024-12-01T18:50:16,843 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6. 2024-12-01T18:50:16,843 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6. 2024-12-01T18:50:16,843 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6. after waiting 0 ms 2024-12-01T18:50:16,843 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6. 2024-12-01T18:50:16,844 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733079012315.6270053f257a61a520e972a1c2aeeef6. 2024-12-01T18:50:16,844 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6270053f257a61a520e972a1c2aeeef6: Waiting for close lock at 1733079016843Disabling compacts and flushes for region at 1733079016843Disabling writes for close at 1733079016843Writing region close event to WAL at 1733079016844 (+1 ms)Closed at 1733079016844 2024-12-01T18:50:16,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741929_1108 (size=175) 2024-12-01T18:50:16,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741929_1108 (size=175) 2024-12-01T18:50:16,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741929_1108 (size=175) 2024-12-01T18:50:16,849 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:50:16,849 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733079016715) 2024-12-01T18:50:16,867 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=443 (was 429) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:38241 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_74830796_22 at /127.0.0.1:37494 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:38241 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_74830796_22 at /127.0.0.1:58230 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_74830796_22 at /127.0.0.1:35440 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=192 (was 209), ProcessCount=11 (was 11), AvailableMemoryMB=3719 (was 3735) 2024-12-01T18:50:16,867 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-01T18:50:16,880 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=443, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=192, ProcessCount=11, AvailableMemoryMB=3718 2024-12-01T18:50:16,880 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-01T18:50:16,895 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:50:16,896 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T18:50:16,897 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T18:50:16,899 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-86032694, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/hregion-86032694, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:16,912 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-86032694/hregion-86032694.1733079016900, exclude list is [], retry=0 2024-12-01T18:50:16,915 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:16,915 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:16,915 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:16,942 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-86032694/hregion-86032694.1733079016900 2024-12-01T18:50:16,942 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921)] 2024-12-01T18:50:16,942 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 64d74f2dc6d825ff3054f3f88a7e5802, NAME => 'testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38241/hbase 2024-12-01T18:50:16,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741931_1110 (size=67) 2024-12-01T18:50:16,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741931_1110 (size=67) 2024-12-01T18:50:16,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741931_1110 (size=67) 2024-12-01T18:50:16,952 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:16,953 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:16,955 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64d74f2dc6d825ff3054f3f88a7e5802 columnFamilyName a 2024-12-01T18:50:16,955 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:16,955 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(327): Store=64d74f2dc6d825ff3054f3f88a7e5802/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:16,955 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:16,956 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64d74f2dc6d825ff3054f3f88a7e5802 columnFamilyName b 2024-12-01T18:50:16,956 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:16,957 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(327): Store=64d74f2dc6d825ff3054f3f88a7e5802/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:16,957 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:16,958 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64d74f2dc6d825ff3054f3f88a7e5802 columnFamilyName c 2024-12-01T18:50:16,958 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:16,959 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(327): Store=64d74f2dc6d825ff3054f3f88a7e5802/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:16,959 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:16,960 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:16,960 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:16,961 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:16,961 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:16,961 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:16,962 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:16,964 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T18:50:16,965 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 64d74f2dc6d825ff3054f3f88a7e5802; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58768783, jitterRate=-0.12427689135074615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:16,965 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 64d74f2dc6d825ff3054f3f88a7e5802: Writing region info on filesystem at 1733079016952Initializing all the Stores at 1733079016953 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079016953Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079016953Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079016953Cleaning up temporary data from old regions at 1733079016961 (+8 ms)Region opened successfully at 1733079016965 (+4 ms) 2024-12-01T18:50:16,965 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 64d74f2dc6d825ff3054f3f88a7e5802, disabling compactions & flushes 2024-12-01T18:50:16,965 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:16,965 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:16,965 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. after waiting 0 ms 2024-12-01T18:50:16,965 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:16,965 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:16,965 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 64d74f2dc6d825ff3054f3f88a7e5802: Waiting for close lock at 1733079016965Disabling compacts and flushes for region at 1733079016965Disabling writes for close at 1733079016965Writing region close event to WAL at 1733079016965Closed at 1733079016965 2024-12-01T18:50:16,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741930_1109 (size=175) 2024-12-01T18:50:16,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741930_1109 (size=175) 2024-12-01T18:50:16,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741930_1109 (size=175) 2024-12-01T18:50:16,970 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:50:16,970 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-86032694:(num 1733079016900) 2024-12-01T18:50:16,971 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:50:16,972 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:16,985 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973, exclude list is [], retry=0 2024-12-01T18:50:16,988 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:16,988 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:16,989 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:17,015 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973 2024-12-01T18:50:17,016 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:50:17,016 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 64d74f2dc6d825ff3054f3f88a7e5802, NAME => 'testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:17,016 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:17,016 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,016 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,018 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,019 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64d74f2dc6d825ff3054f3f88a7e5802 columnFamilyName a 2024-12-01T18:50:17,019 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:17,019 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(327): Store=64d74f2dc6d825ff3054f3f88a7e5802/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:17,019 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,020 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64d74f2dc6d825ff3054f3f88a7e5802 columnFamilyName b 2024-12-01T18:50:17,020 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:17,021 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(327): Store=64d74f2dc6d825ff3054f3f88a7e5802/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:17,021 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,021 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64d74f2dc6d825ff3054f3f88a7e5802 columnFamilyName c 2024-12-01T18:50:17,021 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:17,022 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(327): Store=64d74f2dc6d825ff3054f3f88a7e5802/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:17,022 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,023 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,023 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,024 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,024 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,025 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:17,026 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,027 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 64d74f2dc6d825ff3054f3f88a7e5802; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71437651, jitterRate=0.06450395286083221}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:17,027 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 64d74f2dc6d825ff3054f3f88a7e5802: Writing region info on filesystem at 1733079017016Initializing all the Stores at 1733079017017 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079017017Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079017017Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079017017Cleaning up temporary data from old regions at 1733079017024 (+7 ms)Region opened successfully at 1733079017027 (+3 ms) 2024-12-01T18:50:17,033 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 64d74f2dc6d825ff3054f3f88a7e5802 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-01T18:50:17,049 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/a/a12cfb231fae474c97aad817800e692a is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733079017027/Put/seqid=0 2024-12-01T18:50:17,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741933_1112 (size=5958) 2024-12-01T18:50:17,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741933_1112 (size=5958) 2024-12-01T18:50:17,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741933_1112 (size=5958) 2024-12-01T18:50:17,057 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/a/a12cfb231fae474c97aad817800e692a 2024-12-01T18:50:17,062 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/a/a12cfb231fae474c97aad817800e692a as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/a/a12cfb231fae474c97aad817800e692a 2024-12-01T18:50:17,067 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/a/a12cfb231fae474c97aad817800e692a, entries=10, sequenceid=13, filesize=5.8 K 2024-12-01T18:50:17,068 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 64d74f2dc6d825ff3054f3f88a7e5802 in 35ms, sequenceid=13, compaction requested=false 2024-12-01T18:50:17,068 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 64d74f2dc6d825ff3054f3f88a7e5802: 2024-12-01T18:50:17,084 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 64d74f2dc6d825ff3054f3f88a7e5802, disabling compactions & flushes 2024-12-01T18:50:17,084 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:17,084 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:17,084 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. after waiting 0 ms 2024-12-01T18:50:17,084 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:17,085 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:17,085 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:17,085 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 64d74f2dc6d825ff3054f3f88a7e5802: Waiting for close lock at 1733079017084Disabling compacts and flushes for region at 1733079017084Disabling writes for close at 1733079017084Writing region close event to WAL at 1733079017085 (+1 ms)Closed at 1733079017085 2024-12-01T18:50:17,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741932_1111 (size=6243) 2024-12-01T18:50:17,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741932_1111 (size=6243) 2024-12-01T18:50:17,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741932_1111 (size=6243) 2024-12-01T18:50:17,104 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973, size=6.1 K (6243bytes) 2024-12-01T18:50:17,104 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973 2024-12-01T18:50:17,105 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973 after 1ms 2024-12-01T18:50:17,132 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:17,132 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973 took 28ms 2024-12-01T18:50:17,134 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973 so closing down 2024-12-01T18:50:17,134 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:50:17,136 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733079016973.temp 2024-12-01T18:50:17,162 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000003-wal.1733079016973.temp 2024-12-01T18:50:17,162 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:50:17,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741934_1113 (size=5575) 2024-12-01T18:50:17,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741934_1113 (size=5575) 2024-12-01T18:50:17,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741934_1113 (size=5575) 2024-12-01T18:50:17,169 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000003-wal.1733079016973.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-01T18:50:17,171 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000003-wal.1733079016973.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000035 2024-12-01T18:50:17,171 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 39 ms; skipped=2; WAL=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-01T18:50:17,171 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973, journal: Splitting hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973, size=6.1 K (6243bytes) at 1733079017104Finishing writing output for hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973 so closing down at 1733079017134 (+30 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000003-wal.1733079016973.temp at 1733079017162 (+28 ms)3 split writer threads finished at 1733079017162Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000003-wal.1733079016973.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733079017169 (+7 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000003-wal.1733079016973.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000035 at 1733079017171 (+2 ms)Processed 32 edits across 1 Regions in 39 ms; skipped=2; WAL=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733079017171 2024-12-01T18:50:17,173 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079016973 to hdfs://localhost:38241/hbase/oldWALs/wal.1733079016973 2024-12-01T18:50:17,174 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000035 2024-12-01T18:50:17,174 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:50:17,176 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:17,189 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176, exclude list is [], retry=0 2024-12-01T18:50:17,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:17,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:17,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:17,230 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176 2024-12-01T18:50:17,231 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:37787:37787),(127.0.0.1/127.0.0.1:40921:40921)] 2024-12-01T18:50:17,231 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 64d74f2dc6d825ff3054f3f88a7e5802, NAME => 'testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802.', STARTKEY => '', ENDKEY => ''} 2024-12-01T18:50:17,231 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:17,231 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,231 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,233 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,234 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64d74f2dc6d825ff3054f3f88a7e5802 columnFamilyName a 2024-12-01T18:50:17,234 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:17,239 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/a/a12cfb231fae474c97aad817800e692a 2024-12-01T18:50:17,239 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(327): Store=64d74f2dc6d825ff3054f3f88a7e5802/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:17,240 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,241 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64d74f2dc6d825ff3054f3f88a7e5802 columnFamilyName b 2024-12-01T18:50:17,241 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:17,241 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(327): Store=64d74f2dc6d825ff3054f3f88a7e5802/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:17,241 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,242 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64d74f2dc6d825ff3054f3f88a7e5802 columnFamilyName c 2024-12-01T18:50:17,242 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:17,243 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(327): Store=64d74f2dc6d825ff3054f3f88a7e5802/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:17,243 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,243 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,244 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,245 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000035 2024-12-01T18:50:17,272 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:17,273 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000035 2024-12-01T18:50:17,274 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 64d74f2dc6d825ff3054f3f88a7e5802 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-01T18:50:17,289 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/b/401f9e5a791943638dffd37e78d46e85 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733079017068/Put/seqid=0 2024-12-01T18:50:17,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741936_1115 (size=5958) 2024-12-01T18:50:17,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741936_1115 (size=5958) 2024-12-01T18:50:17,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741936_1115 (size=5958) 2024-12-01T18:50:17,296 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/b/401f9e5a791943638dffd37e78d46e85 2024-12-01T18:50:17,315 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/c/005aadc49dff471ea7fa8e0852ed7fcb is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733079017075/Put/seqid=0 2024-12-01T18:50:17,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741937_1116 (size=5958) 2024-12-01T18:50:17,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741937_1116 (size=5958) 2024-12-01T18:50:17,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741937_1116 (size=5958) 2024-12-01T18:50:17,323 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/c/005aadc49dff471ea7fa8e0852ed7fcb 2024-12-01T18:50:17,329 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/b/401f9e5a791943638dffd37e78d46e85 as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/b/401f9e5a791943638dffd37e78d46e85 2024-12-01T18:50:17,333 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/b/401f9e5a791943638dffd37e78d46e85, entries=10, sequenceid=35, filesize=5.8 K 2024-12-01T18:50:17,334 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/c/005aadc49dff471ea7fa8e0852ed7fcb as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/c/005aadc49dff471ea7fa8e0852ed7fcb 2024-12-01T18:50:17,339 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/c/005aadc49dff471ea7fa8e0852ed7fcb, entries=10, sequenceid=35, filesize=5.8 K 2024-12-01T18:50:17,339 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 64d74f2dc6d825ff3054f3f88a7e5802 in 65ms, sequenceid=35, compaction requested=false; wal=null 2024-12-01T18:50:17,339 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000035 2024-12-01T18:50:17,340 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,340 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,341 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:17,342 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:17,344 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-01T18:50:17,345 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 64d74f2dc6d825ff3054f3f88a7e5802; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74655550, jitterRate=0.1124543845653534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:17,345 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 64d74f2dc6d825ff3054f3f88a7e5802: Writing region info on filesystem at 1733079017232Initializing all the Stores at 1733079017233 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079017233Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079017233Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079017233Obtaining lock to block concurrent updates at 1733079017274 (+41 ms)Preparing flush snapshotting stores in 64d74f2dc6d825ff3054f3f88a7e5802 at 1733079017274Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733079017274Flushing stores of testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. at 1733079017274Flushing 64d74f2dc6d825ff3054f3f88a7e5802/b: creating writer at 1733079017274Flushing 64d74f2dc6d825ff3054f3f88a7e5802/b: appending metadata at 1733079017288 (+14 ms)Flushing 64d74f2dc6d825ff3054f3f88a7e5802/b: closing flushed file at 1733079017288Flushing 64d74f2dc6d825ff3054f3f88a7e5802/c: creating writer at 1733079017300 (+12 ms)Flushing 64d74f2dc6d825ff3054f3f88a7e5802/c: appending metadata at 1733079017315 (+15 ms)Flushing 64d74f2dc6d825ff3054f3f88a7e5802/c: closing flushed file at 1733079017315Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21e616e7: reopening flushed file at 1733079017328 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1754bc9e: reopening flushed file at 1733079017333 (+5 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 64d74f2dc6d825ff3054f3f88a7e5802 in 65ms, sequenceid=35, compaction requested=false; wal=null at 1733079017339 (+6 ms)Cleaning up temporary data from old regions at 1733079017340 (+1 ms)Region opened successfully at 1733079017345 (+5 ms) 2024-12-01T18:50:17,426 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176, size=0 (0bytes) 2024-12-01T18:50:17,426 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176 might be still open, length is 0 2024-12-01T18:50:17,426 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176 2024-12-01T18:50:17,427 WARN [IPC Server handler 2 on default port 38241 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-01T18:50:17,427 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176 after 1ms 2024-12-01T18:50:19,060 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:35474 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:35899:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35474 dst: /127.0.0.1:35899 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35899 remote=/127.0.0.1:35474]. Total timeout mills is 60000, 58314 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:19,061 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:58282 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41037:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58282 dst: /127.0.0.1:41037 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:19,061 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_495988095_22 at /127.0.0.1:37548 [Receiving block BP-1540968591-172.17.0.2-1733078978096:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:43787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37548 dst: /127.0.0.1:43787 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:19,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741935_1117 (size=5567) 2024-12-01T18:50:19,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741935_1117 (size=5567) 2024-12-01T18:50:21,427 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176 after 4001ms 2024-12-01T18:50:21,456 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:21,456 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176 took 4030ms 2024-12-01T18:50:21,458 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176; continuing. 2024-12-01T18:50:21,458 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176 so closing down 2024-12-01T18:50:21,458 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T18:50:21,459 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733079017176.temp 2024-12-01T18:50:21,485 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000037-wal.1733079017176.temp 2024-12-01T18:50:21,485 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T18:50:21,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741938_1118 (size=5575) 2024-12-01T18:50:21,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741938_1118 (size=5575) 2024-12-01T18:50:21,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741938_1118 (size=5575) 2024-12-01T18:50:21,494 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000037-wal.1733079017176.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-01T18:50:21,496 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000037-wal.1733079017176.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000066 2024-12-01T18:50:21,496 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176, size=0, length=0, corrupted=false, cancelled=false 2024-12-01T18:50:21,496 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176, journal: Splitting hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176, size=0 (0bytes) at 1733079017426Finishing writing output for hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176 so closing down at 1733079021458 (+4032 ms)Creating recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000037-wal.1733079017176.temp at 1733079021485 (+27 ms)3 split writer threads finished at 1733079021485Closed recovered edits writer path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000037-wal.1733079017176.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733079021494 (+9 ms)Rename recovered edits hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000037-wal.1733079017176.temp to hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000066 at 1733079021496 (+2 ms)Processed 30 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176, size=0, length=0, corrupted=false, cancelled=false at 1733079021496 2024-12-01T18:50:21,497 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176 to hdfs://localhost:38241/hbase/oldWALs/wal.1733079017176 2024-12-01T18:50:21,498 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000066 2024-12-01T18:50:21,498 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T18:50:21,500 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38241/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894, archiveDir=hdfs://localhost:38241/hbase/oldWALs, maxLogs=32 2024-12-01T18:50:21,512 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079021500, exclude list is [], retry=0 2024-12-01T18:50:21,514 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35899,DS-34a4119c-ae8a-4ee9-8e77-13b08cf7e84f,DISK] 2024-12-01T18:50:21,515 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43787,DS-87ccdf44-2281-4f1f-961d-d22bdcf6ec98,DISK] 2024-12-01T18:50:21,515 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41037,DS-4811a14f-9cd8-41a9-be38-25d01bc29430,DISK] 2024-12-01T18:50:21,521 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-01T18:50:21,521 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-01T18:50:21,522 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-01T18:50:21,522 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-01T18:50:21,541 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079021500 2024-12-01T18:50:21,541 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601),(127.0.0.1/127.0.0.1:40921:40921),(127.0.0.1/127.0.0.1:37787:37787)] 2024-12-01T18:50:21,542 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T18:50:21,543 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:21,544 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64d74f2dc6d825ff3054f3f88a7e5802 columnFamilyName a 2024-12-01T18:50:21,544 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:21,549 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/a/a12cfb231fae474c97aad817800e692a 2024-12-01T18:50:21,549 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(327): Store=64d74f2dc6d825ff3054f3f88a7e5802/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:21,549 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:21,550 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64d74f2dc6d825ff3054f3f88a7e5802 columnFamilyName b 2024-12-01T18:50:21,550 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:21,554 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/b/401f9e5a791943638dffd37e78d46e85 2024-12-01T18:50:21,554 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(327): Store=64d74f2dc6d825ff3054f3f88a7e5802/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:21,554 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:21,555 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64d74f2dc6d825ff3054f3f88a7e5802 columnFamilyName c 2024-12-01T18:50:21,555 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T18:50:21,559 DEBUG [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/c/005aadc49dff471ea7fa8e0852ed7fcb 2024-12-01T18:50:21,559 INFO [StoreOpener-64d74f2dc6d825ff3054f3f88a7e5802-1 {}] regionserver.HStore(327): Store=64d74f2dc6d825ff3054f3f88a7e5802/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T18:50:21,559 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:21,559 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:21,560 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:21,561 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000066 2024-12-01T18:50:21,587 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T18:50:21,592 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000066 2024-12-01T18:50:21,592 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 64d74f2dc6d825ff3054f3f88a7e5802 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-01T18:50:21,606 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/a/8dbcdfbddec84e9ebd9aa5991ac4da94 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733079017351/Put/seqid=0 2024-12-01T18:50:21,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741940_1120 (size=5958) 2024-12-01T18:50:21,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741940_1120 (size=5958) 2024-12-01T18:50:21,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741940_1120 (size=5958) 2024-12-01T18:50:21,612 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/a/8dbcdfbddec84e9ebd9aa5991ac4da94 2024-12-01T18:50:21,630 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/b/906bc78112c744e59fe381035e653c27 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733079017359/Put/seqid=0 2024-12-01T18:50:21,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741941_1121 (size=5958) 2024-12-01T18:50:21,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741941_1121 (size=5958) 2024-12-01T18:50:21,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741941_1121 (size=5958) 2024-12-01T18:50:21,636 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/b/906bc78112c744e59fe381035e653c27 2024-12-01T18:50:21,654 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/c/3a8a622b7e4d4b30b4cf52be03ceaa5b is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733079017366/Put/seqid=0 2024-12-01T18:50:21,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741942_1122 (size=5958) 2024-12-01T18:50:21,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741942_1122 (size=5958) 2024-12-01T18:50:21,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741942_1122 (size=5958) 2024-12-01T18:50:21,661 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/c/3a8a622b7e4d4b30b4cf52be03ceaa5b 2024-12-01T18:50:21,666 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/a/8dbcdfbddec84e9ebd9aa5991ac4da94 as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/a/8dbcdfbddec84e9ebd9aa5991ac4da94 2024-12-01T18:50:21,670 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/a/8dbcdfbddec84e9ebd9aa5991ac4da94, entries=10, sequenceid=66, filesize=5.8 K 2024-12-01T18:50:21,670 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/b/906bc78112c744e59fe381035e653c27 as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/b/906bc78112c744e59fe381035e653c27 2024-12-01T18:50:21,675 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/b/906bc78112c744e59fe381035e653c27, entries=10, sequenceid=66, filesize=5.8 K 2024-12-01T18:50:21,675 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/.tmp/c/3a8a622b7e4d4b30b4cf52be03ceaa5b as hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/c/3a8a622b7e4d4b30b4cf52be03ceaa5b 2024-12-01T18:50:21,679 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/c/3a8a622b7e4d4b30b4cf52be03ceaa5b, entries=10, sequenceid=66, filesize=5.8 K 2024-12-01T18:50:21,679 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 64d74f2dc6d825ff3054f3f88a7e5802 in 87ms, sequenceid=66, compaction requested=false; wal=null 2024-12-01T18:50:21,680 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/0000000000000000066 2024-12-01T18:50:21,681 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:21,681 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:21,681 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T18:50:21,682 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 64d74f2dc6d825ff3054f3f88a7e5802 2024-12-01T18:50:21,684 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/hbase/data/default/testReplayEditsWrittenViaHRegion/64d74f2dc6d825ff3054f3f88a7e5802/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-01T18:50:21,685 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 64d74f2dc6d825ff3054f3f88a7e5802; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62501602, jitterRate=-0.06865355372428894}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T18:50:21,685 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 64d74f2dc6d825ff3054f3f88a7e5802: Writing region info on filesystem at 1733079021542Initializing all the Stores at 1733079021543 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079021543Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079021543Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733079021543Obtaining lock to block concurrent updates at 1733079021592 (+49 ms)Preparing flush snapshotting stores in 64d74f2dc6d825ff3054f3f88a7e5802 at 1733079021592Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733079021592Flushing stores of testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. at 1733079021592Flushing 64d74f2dc6d825ff3054f3f88a7e5802/a: creating writer at 1733079021592Flushing 64d74f2dc6d825ff3054f3f88a7e5802/a: appending metadata at 1733079021605 (+13 ms)Flushing 64d74f2dc6d825ff3054f3f88a7e5802/a: closing flushed file at 1733079021605Flushing 64d74f2dc6d825ff3054f3f88a7e5802/b: creating writer at 1733079021617 (+12 ms)Flushing 64d74f2dc6d825ff3054f3f88a7e5802/b: appending metadata at 1733079021630 (+13 ms)Flushing 64d74f2dc6d825ff3054f3f88a7e5802/b: closing flushed file at 1733079021630Flushing 64d74f2dc6d825ff3054f3f88a7e5802/c: creating writer at 1733079021640 (+10 ms)Flushing 64d74f2dc6d825ff3054f3f88a7e5802/c: appending metadata at 1733079021654 (+14 ms)Flushing 64d74f2dc6d825ff3054f3f88a7e5802/c: closing flushed file at 1733079021654Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@588acab: reopening flushed file at 1733079021665 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a3da741: reopening flushed file at 1733079021670 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@771aa93e: reopening flushed file at 1733079021675 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 64d74f2dc6d825ff3054f3f88a7e5802 in 87ms, sequenceid=66, compaction requested=false; wal=null at 1733079021679 (+4 ms)Cleaning up temporary data from old regions at 1733079021681 (+2 ms)Region opened successfully at 1733079021685 (+4 ms) 2024-12-01T18:50:21,697 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 64d74f2dc6d825ff3054f3f88a7e5802, disabling compactions & flushes 2024-12-01T18:50:21,697 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:21,697 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:21,697 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. after waiting 0 ms 2024-12-01T18:50:21,697 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:21,698 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733079016895.64d74f2dc6d825ff3054f3f88a7e5802. 2024-12-01T18:50:21,698 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 64d74f2dc6d825ff3054f3f88a7e5802: Waiting for close lock at 1733079021697Disabling compacts and flushes for region at 1733079021697Disabling writes for close at 1733079021697Writing region close event to WAL at 1733079021698 (+1 ms)Closed at 1733079021698 2024-12-01T18:50:21,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741939_1119 (size=175) 2024-12-01T18:50:21,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741939_1119 (size=175) 2024-12-01T18:50:21,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741939_1119 (size=175) 2024-12-01T18:50:21,701 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079021500 not finished, retry = 0 2024-12-01T18:50:21,804 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T18:50:21,804 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733079021500) 2024-12-01T18:50:21,818 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=447 (was 443) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:38241 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:38241 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1990985213_22 at /127.0.0.1:58640 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1990985213_22 at /127.0.0.1:50352 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1990985213_22 at /127.0.0.1:45146 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1412 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=185 (was 192), ProcessCount=11 (was 11), AvailableMemoryMB=3709 (was 3718) 2024-12-01T18:50:21,818 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1412 is superior to 1024 2024-12-01T18:50:21,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-01T18:50:21,819 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T18:50:21,819 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T18:50:21,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T18:50:21,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T18:50:21,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T18:50:21,820 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-01T18:50:21,820 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1885054661, stopped=false 2024-12-01T18:50:21,820 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9ec37ae3355e,40221,1733078981110 2024-12-01T18:50:21,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T18:50:21,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T18:50:21,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T18:50:21,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:50:21,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:50:21,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:50:21,822 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T18:50:21,822 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T18:50:21,822 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T18:50:21,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T18:50:21,823 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9ec37ae3355e,37405,1733078981806' ***** 2024-12-01T18:50:21,823 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T18:50:21,823 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9ec37ae3355e,35069,1733078981915' ***** 2024-12-01T18:50:21,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T18:50:21,823 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T18:50:21,823 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T18:50:21,823 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T18:50:21,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T18:50:21,823 INFO [RS:0;9ec37ae3355e:37405 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T18:50:21,823 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T18:50:21,823 INFO [RS:1;9ec37ae3355e:35069 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T18:50:21,823 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T18:50:21,823 INFO [RS:0;9ec37ae3355e:37405 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T18:50:21,823 INFO [RS:1;9ec37ae3355e:35069 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T18:50:21,823 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:21,823 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(959): stopping server 9ec37ae3355e,35069,1733078981915 2024-12-01T18:50:21,824 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(959): stopping server 9ec37ae3355e,37405,1733078981806 2024-12-01T18:50:21,824 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T18:50:21,824 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T18:50:21,824 INFO [RS:0;9ec37ae3355e:37405 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9ec37ae3355e:37405. 2024-12-01T18:50:21,824 INFO [RS:1;9ec37ae3355e:35069 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9ec37ae3355e:35069. 2024-12-01T18:50:21,824 DEBUG [RS:0;9ec37ae3355e:37405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T18:50:21,824 DEBUG [RS:0;9ec37ae3355e:37405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T18:50:21,824 DEBUG [RS:1;9ec37ae3355e:35069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T18:50:21,824 DEBUG [RS:1;9ec37ae3355e:35069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T18:50:21,824 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0ff3eaf0745a3ffb1311b6b12692fba4, disabling compactions & flushes 2024-12-01T18:50:21,824 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-01T18:50:21,824 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:50:21,824 DEBUG [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(1325): Online Regions={0ff3eaf0745a3ffb1311b6b12692fba4=testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4.} 2024-12-01T18:50:21,824 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:50:21,824 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T18:50:21,824 DEBUG [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(1351): Waiting on 0ff3eaf0745a3ffb1311b6b12692fba4 2024-12-01T18:50:21,824 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. after waiting 0 ms 2024-12-01T18:50:21,824 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T18:50:21,824 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:50:21,824 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T18:50:21,824 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T18:50:21,825 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-01T18:50:21,825 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T18:50:21,825 DEBUG [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-01T18:50:21,825 DEBUG [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-01T18:50:21,825 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T18:50:21,825 INFO [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T18:50:21,825 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T18:50:21,825 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T18:50:21,825 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T18:50:21,825 INFO [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-01T18:50:21,829 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0ff3eaf0745a3ffb1311b6b12692fba4/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-01T18:50:21,829 INFO [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:50:21,830 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0ff3eaf0745a3ffb1311b6b12692fba4: Waiting for close lock at 1733079021824Running coprocessor pre-close hooks at 1733079021824Disabling compacts and flushes for region at 1733079021824Disabling writes for close at 1733079021824Writing region close event to WAL at 1733079021825 (+1 ms)Running coprocessor post-close hooks at 1733079021829 (+4 ms)Closed at 1733079021829 2024-12-01T18:50:21,830 DEBUG [RS_CLOSE_REGION-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4. 2024-12-01T18:50:21,830 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/.tmp/info/d9db0ff5b60d4c3696d8ac8143dc49af is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733078997281.0ff3eaf0745a3ffb1311b6b12692fba4./info:regioninfo/1733079001290/Put/seqid=0 2024-12-01T18:50:21,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741943_1123 (size=6778) 2024-12-01T18:50:21,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741943_1123 (size=6778) 2024-12-01T18:50:21,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741943_1123 (size=6778) 2024-12-01T18:50:21,836 INFO [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/.tmp/info/d9db0ff5b60d4c3696d8ac8143dc49af 2024-12-01T18:50:21,842 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/.tmp/info/d9db0ff5b60d4c3696d8ac8143dc49af as hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/info/d9db0ff5b60d4c3696d8ac8143dc49af 2024-12-01T18:50:21,846 INFO [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/info/d9db0ff5b60d4c3696d8ac8143dc49af, entries=8, sequenceid=23, filesize=6.6 K 2024-12-01T18:50:21,847 INFO [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 22ms, sequenceid=23, compaction requested=false 2024-12-01T18:50:21,851 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-01T18:50:21,852 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T18:50:21,852 INFO [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T18:50:21,852 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733079021825Running coprocessor pre-close hooks at 1733079021825Disabling compacts and flushes for region at 1733079021825Disabling writes for close at 1733079021825Obtaining lock to block concurrent updates at 1733079021825Preparing flush snapshotting stores in 1588230740 at 1733079021825Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733079021825Flushing stores of hbase:meta,,1.1588230740 at 1733079021826 (+1 ms)Flushing 1588230740/info: creating writer at 1733079021826Flushing 1588230740/info: appending metadata at 1733079021829 (+3 ms)Flushing 1588230740/info: closing flushed file at 1733079021829Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4444b7c: reopening flushed file at 1733079021841 (+12 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 22ms, sequenceid=23, compaction requested=false at 1733079021847 (+6 ms)Writing region close event to WAL at 1733079021848 (+1 ms)Running coprocessor post-close hooks at 1733079021852 (+4 ms)Closed at 1733079021852 2024-12-01T18:50:21,852 DEBUG [RS_CLOSE_META-regionserver/9ec37ae3355e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T18:50:21,864 INFO [regionserver/9ec37ae3355e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T18:50:21,865 INFO [regionserver/9ec37ae3355e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T18:50:22,024 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(976): stopping server 9ec37ae3355e,37405,1733078981806; all regions closed. 2024-12-01T18:50:22,025 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(976): stopping server 9ec37ae3355e,35069,1733078981915; all regions closed. 2024-12-01T18:50:22,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741834_1010 (size=3621) 2024-12-01T18:50:22,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741834_1010 (size=3621) 2024-12-01T18:50:22,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741834_1010 (size=3621) 2024-12-01T18:50:22,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741893_1071 (size=2625) 2024-12-01T18:50:22,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741893_1071 (size=2625) 2024-12-01T18:50:22,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741893_1071 (size=2625) 2024-12-01T18:50:22,030 DEBUG [RS:0;9ec37ae3355e:37405 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/oldWALs 2024-12-01T18:50:22,030 INFO [RS:0;9ec37ae3355e:37405 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ec37ae3355e%2C37405%2C1733078981806:(num 1733078983296) 2024-12-01T18:50:22,030 DEBUG [RS:0;9ec37ae3355e:37405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T18:50:22,030 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T18:50:22,030 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T18:50:22,030 DEBUG [RS:1;9ec37ae3355e:35069 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/oldWALs 2024-12-01T18:50:22,030 INFO [RS:1;9ec37ae3355e:35069 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ec37ae3355e%2C35069%2C1733078981915.meta:.meta(num 1733079000373) 2024-12-01T18:50:22,031 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.ChoreService(370): Chore service for: regionserver/9ec37ae3355e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T18:50:22,031 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T18:50:22,031 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T18:50:22,031 INFO [regionserver/9ec37ae3355e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T18:50:22,031 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T18:50:22,031 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T18:50:22,031 INFO [RS:0;9ec37ae3355e:37405 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37405 2024-12-01T18:50:22,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741835_1011 (size=175) 2024-12-01T18:50:22,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741835_1011 (size=175) 2024-12-01T18:50:22,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741835_1011 (size=175) 2024-12-01T18:50:22,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9ec37ae3355e,37405,1733078981806 2024-12-01T18:50:22,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T18:50:22,034 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T18:50:22,035 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9ec37ae3355e,37405,1733078981806] 2024-12-01T18:50:22,036 DEBUG [RS:1;9ec37ae3355e:35069 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/oldWALs 2024-12-01T18:50:22,036 INFO [RS:1;9ec37ae3355e:35069 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ec37ae3355e%2C35069%2C1733078981915:(num 1733078983296) 2024-12-01T18:50:22,036 DEBUG [RS:1;9ec37ae3355e:35069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T18:50:22,036 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T18:50:22,036 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T18:50:22,036 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.ChoreService(370): Chore service for: regionserver/9ec37ae3355e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-01T18:50:22,036 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T18:50:22,036 INFO [regionserver/9ec37ae3355e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T18:50:22,036 INFO [RS:1;9ec37ae3355e:35069 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35069 2024-12-01T18:50:22,038 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9ec37ae3355e,37405,1733078981806 already deleted, retry=false 2024-12-01T18:50:22,038 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9ec37ae3355e,37405,1733078981806 expired; onlineServers=1 2024-12-01T18:50:22,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9ec37ae3355e,35069,1733078981915 2024-12-01T18:50:22,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T18:50:22,039 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T18:50:22,040 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9ec37ae3355e,35069,1733078981915] 2024-12-01T18:50:22,041 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9ec37ae3355e,35069,1733078981915 already deleted, retry=false 2024-12-01T18:50:22,041 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9ec37ae3355e,35069,1733078981915 expired; onlineServers=0 2024-12-01T18:50:22,041 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9ec37ae3355e,40221,1733078981110' ***** 2024-12-01T18:50:22,041 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-01T18:50:22,041 INFO [M:0;9ec37ae3355e:40221 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T18:50:22,041 INFO [M:0;9ec37ae3355e:40221 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T18:50:22,042 DEBUG [M:0;9ec37ae3355e:40221 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-01T18:50:22,042 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-01T18:50:22,042 DEBUG [M:0;9ec37ae3355e:40221 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-01T18:50:22,042 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster-HFileCleaner.large.0-1733078982974 {}] cleaner.HFileCleaner(306): Exit Thread[master/9ec37ae3355e:0:becomeActiveMaster-HFileCleaner.large.0-1733078982974,5,FailOnTimeoutGroup] 2024-12-01T18:50:22,042 DEBUG [master/9ec37ae3355e:0:becomeActiveMaster-HFileCleaner.small.0-1733078982977 {}] cleaner.HFileCleaner(306): Exit Thread[master/9ec37ae3355e:0:becomeActiveMaster-HFileCleaner.small.0-1733078982977,5,FailOnTimeoutGroup] 2024-12-01T18:50:22,042 INFO [M:0;9ec37ae3355e:40221 {}] hbase.ChoreService(370): Chore service for: master/9ec37ae3355e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-01T18:50:22,042 INFO [M:0;9ec37ae3355e:40221 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T18:50:22,042 DEBUG [M:0;9ec37ae3355e:40221 {}] master.HMaster(1795): Stopping service threads 2024-12-01T18:50:22,042 INFO [M:0;9ec37ae3355e:40221 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-01T18:50:22,043 INFO [M:0;9ec37ae3355e:40221 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T18:50:22,043 INFO [M:0;9ec37ae3355e:40221 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-01T18:50:22,043 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-01T18:50:22,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-01T18:50:22,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T18:50:22,045 DEBUG [M:0;9ec37ae3355e:40221 {}] zookeeper.ZKUtil(347): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-01T18:50:22,045 WARN [M:0;9ec37ae3355e:40221 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-01T18:50:22,046 INFO [M:0;9ec37ae3355e:40221 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/.lastflushedseqids 2024-12-01T18:50:22,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741944_1124 (size=119) 2024-12-01T18:50:22,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741944_1124 (size=119) 2024-12-01T18:50:22,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741944_1124 (size=119) 2024-12-01T18:50:22,059 INFO [M:0;9ec37ae3355e:40221 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-01T18:50:22,059 INFO [M:0;9ec37ae3355e:40221 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-01T18:50:22,059 DEBUG [M:0;9ec37ae3355e:40221 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T18:50:22,059 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T18:50:22,059 DEBUG [M:0;9ec37ae3355e:40221 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T18:50:22,060 DEBUG [M:0;9ec37ae3355e:40221 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T18:50:22,060 DEBUG [M:0;9ec37ae3355e:40221 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T18:50:22,060 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.38 KB heapSize=102.70 KB 2024-12-01T18:50:22,075 DEBUG [M:0;9ec37ae3355e:40221 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8385913a76e24435977962a5f9c08f8c is 82, key is hbase:meta,,1/info:regioninfo/1733079000606/Put/seqid=0 2024-12-01T18:50:22,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741945_1125 (size=6063) 2024-12-01T18:50:22,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741945_1125 (size=6063) 2024-12-01T18:50:22,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741945_1125 (size=6063) 2024-12-01T18:50:22,081 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8385913a76e24435977962a5f9c08f8c 2024-12-01T18:50:22,099 DEBUG [M:0;9ec37ae3355e:40221 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/06216a03d0da4b049d5f5d5d04399d51 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733078997700/Put/seqid=0 2024-12-01T18:50:22,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741946_1126 (size=7906) 2024-12-01T18:50:22,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741946_1126 (size=7906) 2024-12-01T18:50:22,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741946_1126 (size=7906) 2024-12-01T18:50:22,106 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.16 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/06216a03d0da4b049d5f5d5d04399d51 2024-12-01T18:50:22,109 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 06216a03d0da4b049d5f5d5d04399d51 2024-12-01T18:50:22,124 DEBUG [M:0;9ec37ae3355e:40221 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c5eda459c6cb47cfbc5dc5cad3651939 is 69, key is 9ec37ae3355e,35069,1733078981915/rs:state/1733078983041/Put/seqid=0 2024-12-01T18:50:22,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741947_1127 (size=5440) 2024-12-01T18:50:22,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741947_1127 (size=5440) 2024-12-01T18:50:22,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741947_1127 (size=5440) 2024-12-01T18:50:22,130 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c5eda459c6cb47cfbc5dc5cad3651939 2024-12-01T18:50:22,135 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c5eda459c6cb47cfbc5dc5cad3651939 2024-12-01T18:50:22,135 DEBUG [M:0;9ec37ae3355e:40221 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8385913a76e24435977962a5f9c08f8c as hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8385913a76e24435977962a5f9c08f8c 2024-12-01T18:50:22,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T18:50:22,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x1016f5c81760001, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T18:50:22,136 INFO [RS:0;9ec37ae3355e:37405 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T18:50:22,136 INFO [RS:0;9ec37ae3355e:37405 {}] regionserver.HRegionServer(1031): Exiting; stopping=9ec37ae3355e,37405,1733078981806; zookeeper connection closed. 2024-12-01T18:50:22,136 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a0b0afd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a0b0afd 2024-12-01T18:50:22,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T18:50:22,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35069-0x1016f5c81760002, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T18:50:22,140 INFO [RS:1;9ec37ae3355e:35069 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T18:50:22,140 INFO [RS:1;9ec37ae3355e:35069 {}] regionserver.HRegionServer(1031): Exiting; stopping=9ec37ae3355e,35069,1733078981915; zookeeper connection closed. 2024-12-01T18:50:22,140 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8385913a76e24435977962a5f9c08f8c, entries=14, sequenceid=207, filesize=5.9 K 2024-12-01T18:50:22,140 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5c4b21dc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5c4b21dc 2024-12-01T18:50:22,141 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-01T18:50:22,141 DEBUG [M:0;9ec37ae3355e:40221 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/06216a03d0da4b049d5f5d5d04399d51 as hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/06216a03d0da4b049d5f5d5d04399d51 2024-12-01T18:50:22,146 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 06216a03d0da4b049d5f5d5d04399d51 2024-12-01T18:50:22,146 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/06216a03d0da4b049d5f5d5d04399d51, entries=21, sequenceid=207, filesize=7.7 K 2024-12-01T18:50:22,147 DEBUG [M:0;9ec37ae3355e:40221 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c5eda459c6cb47cfbc5dc5cad3651939 as hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c5eda459c6cb47cfbc5dc5cad3651939 2024-12-01T18:50:22,151 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c5eda459c6cb47cfbc5dc5cad3651939 2024-12-01T18:50:22,151 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38241/user/jenkins/test-data/62253768-3791-7f38-868b-977476e45018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c5eda459c6cb47cfbc5dc5cad3651939, entries=3, sequenceid=207, filesize=5.3 K 2024-12-01T18:50:22,152 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=207, compaction requested=false 2024-12-01T18:50:22,153 INFO [M:0;9ec37ae3355e:40221 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T18:50:22,153 DEBUG [M:0;9ec37ae3355e:40221 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733079022059Disabling compacts and flushes for region at 1733079022059Disabling writes for close at 1733079022060 (+1 ms)Obtaining lock to block concurrent updates at 1733079022060Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733079022060Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85386, getHeapSize=105096, getOffHeapSize=0, getCellsCount=248 at 1733079022060Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733079022061 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733079022061Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733079022075 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733079022075Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733079022085 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733079022099 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733079022099Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733079022110 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733079022123 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733079022123Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b514596: reopening flushed file at 1733079022135 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@186c9c4e: reopening flushed file at 1733079022140 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@507a3081: reopening flushed file at 1733079022146 (+6 ms)Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=207, compaction requested=false at 1733079022152 (+6 ms)Writing region close event to WAL at 1733079022153 (+1 ms)Closed at 1733079022153 2024-12-01T18:50:22,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35899 is added to blk_1073741830_1006 (size=102150) 2024-12-01T18:50:22,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41037 is added to blk_1073741830_1006 (size=102150) 2024-12-01T18:50:22,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43787 is added to blk_1073741830_1006 (size=102150) 2024-12-01T18:50:22,156 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T18:50:22,156 INFO [M:0;9ec37ae3355e:40221 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-01T18:50:22,156 INFO [M:0;9ec37ae3355e:40221 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40221 2024-12-01T18:50:22,156 INFO [M:0;9ec37ae3355e:40221 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T18:50:22,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T18:50:22,258 INFO [M:0;9ec37ae3355e:40221 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T18:50:22,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40221-0x1016f5c81760000, quorum=127.0.0.1:55741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T18:50:22,264 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079011316 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079011316 (inode 16688) Holder DFSClient_NONMAPREDUCE_495988095_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733079001996/wal.1733079011316 (inode 16688) Holder DFSClient_NONMAPREDUCE_495988095_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-01T18:50:22,266 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001791 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001791 (inode 16665) Holder DFSClient_NONMAPREDUCE_495988095_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733079001372/wal.1733079001791 (inode 16665) Holder DFSClient_NONMAPREDUCE_495988095_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-01T18:50:22,267 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733078992731/wal.1733078992864 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:22,269 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079012119 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079012119 (inode 16714) Holder DFSClient_NONMAPREDUCE_495988095_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733079011555/wal.1733079012119 (inode 16714) Holder DFSClient_NONMAPREDUCE_495988095_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-01T18:50:22,269 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733079012314/wal.1733079012402 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:22,270 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733079016894/wal.1733079017176 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:22,270 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733078985627/wal.1733078985720 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T18:50:22,272 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal.1733078985446 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal.1733078985446 (inode 16485) Holder DFSClient_NONMAPREDUCE_495988095_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733078985067/wal.1733078985446 (inode 16485) Holder DFSClient_NONMAPREDUCE_495988095_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-01T18:50:22,274 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal.1733078984790 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal.1733078984790 (inode 16462) Holder DFSClient_NONMAPREDUCE_495988095_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733078984328/wal.1733078984790 (inode 16462) Holder DFSClient_NONMAPREDUCE_495988095_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-01T18:50:22,278 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T18:50:22,280 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T18:50:22,281 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T18:50:22,281 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T18:50:22,281 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/hadoop.log.dir/,STOPPED} 2024-12-01T18:50:22,283 WARN [BP-1540968591-172.17.0.2-1733078978096 heartbeating to localhost/127.0.0.1:38241 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-01T18:50:22,283 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-01T18:50:22,283 WARN [BP-1540968591-172.17.0.2-1733078978096 heartbeating to localhost/127.0.0.1:38241 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1540968591-172.17.0.2-1733078978096 (Datanode Uuid a1e2453e-a549-4712-98fd-e539af60652c) service to localhost/127.0.0.1:38241 2024-12-01T18:50:22,283 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-01T18:50:22,285 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data5/current/BP-1540968591-172.17.0.2-1733078978096 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T18:50:22,285 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data6/current/BP-1540968591-172.17.0.2-1733078978096 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T18:50:22,285 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-01T18:50:22,287 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T18:50:22,287 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T18:50:22,288 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T18:50:22,288 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T18:50:22,288 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/hadoop.log.dir/,STOPPED} 2024-12-01T18:50:22,289 WARN [BP-1540968591-172.17.0.2-1733078978096 heartbeating to localhost/127.0.0.1:38241 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-01T18:50:22,289 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-01T18:50:22,289 WARN [BP-1540968591-172.17.0.2-1733078978096 heartbeating to localhost/127.0.0.1:38241 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1540968591-172.17.0.2-1733078978096 (Datanode Uuid 06542ef0-158e-42de-b8da-c2e2b0c303cd) service to localhost/127.0.0.1:38241 2024-12-01T18:50:22,289 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-01T18:50:22,290 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data3/current/BP-1540968591-172.17.0.2-1733078978096 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T18:50:22,290 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data4/current/BP-1540968591-172.17.0.2-1733078978096 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T18:50:22,290 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-01T18:50:22,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T18:50:22,297 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T18:50:22,297 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T18:50:22,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T18:50:22,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/hadoop.log.dir/,STOPPED} 2024-12-01T18:50:22,298 WARN [BP-1540968591-172.17.0.2-1733078978096 heartbeating to localhost/127.0.0.1:38241 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-01T18:50:22,298 WARN [BP-1540968591-172.17.0.2-1733078978096 heartbeating to localhost/127.0.0.1:38241 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1540968591-172.17.0.2-1733078978096 (Datanode Uuid da0abfe6-180d-4c93-825e-ed7b0c31716a) service to localhost/127.0.0.1:38241 2024-12-01T18:50:22,298 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-01T18:50:22,298 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-01T18:50:22,299 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data1/current/BP-1540968591-172.17.0.2-1733078978096 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T18:50:22,299 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/cluster_3f8d0923-eb04-d395-d5d1-3f827a04ff8c/data/data2/current/BP-1540968591-172.17.0.2-1733078978096 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T18:50:22,299 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-01T18:50:22,306 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-01T18:50:22,307 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T18:50:22,307 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T18:50:22,307 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T18:50:22,307 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ccba56-2f68-eeef-8e97-f7940b1b10ff/hadoop.log.dir/,STOPPED} 2024-12-01T18:50:22,316 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-01T18:50:22,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down